delete repartition

linear_regression.py

@@ -114,7 +114,6 @@ def process_files(folder: str):
     structured_data = structured_data.withColumn("date", F.to_date(F.col("yearmoda"), "yyyyMMdd")).drop("yearmoda")
 
     medians_pre_df = structured_data.select("stn", "temp", F.year("date").alias("year"))
-    medians_pre_df = medians_pre_df.repartition("stn", "year")
     medians_per_station_per_year = medians_pre_df.groupBy("stn", "year").agg(F.median("temp").alias("median_temp"))
     medians_per_year = medians_per_station_per_year.groupBy("year").agg(F.median("median_temp").alias("median"))
     medians_per_year = medians_per_year.select("year", "median")