Chunked decryption, resume & selection fixes

Implement chunked decryption of .m4s segments (avoid loading entire stream into memory) by changing decrypt() to accept a segment directory, process ~1GB chunks with mp4ff-decrypt, and concatenate results. Add resume support in segment downloader by skipping already downloaded .m4s files and initializing successful counter accordingly. Improve manifest handling: gracefully handle missing AVC/HEVC tracks, bruteforce ticks conditionally, and mark representations for removal when no valid tick is found. Add fallback unfiltered selection in select_track when filtered lookup yields no candidates. Replace naive UTC offset logic with explicit Europe/Paris timezone utilities and adjust related time conversion signatures. Bump project version to 1.0.

utils/input.py

@@ -439,8 +439,16 @@ def select_track(content_dict, quality_spec, content_type):
         candidates.extend(tracks)
 
     if not candidates:
-        logger.warning("No %s track found for '%s'.", content_type, quality_spec)
-        return None
+        if filter_part:
+            logger.warning(
+                "No %s track found for '%s'. Falling back to unfiltered selection.",
+                content_type, quality_spec
+            )
+            for key, tracks in content_dict.items():
+                candidates.extend(tracks)
+        if not candidates:
+            logger.warning("No %s track found for '%s'.", content_type, quality_spec)
+            return None
 
     if pref == "best":
         selected = max(candidates, key=lambda x: x["bandwidth"])

utils/stream.py

@@ -456,8 +456,19 @@ async def save_segments(
 
     # Build list of all segments to download
     segments_to_download = [(start_tick + i * duration, i) for i in range(rep_nb)]
+
+    # In case of resuming, check which segments are already downloaded
+    already_downloaded = [
+        int(f.split(".")[0])
+        for f in os.listdir(f"{output_folder}/segments_{track_id}")
+        if f.endswith(".m4s") and f.split(".")[0].isdigit()
+    ]
+    segments_to_download = [
+        (tick, rep) for tick, rep in segments_to_download if tick not in already_downloaded
+    ]
+
     retry_list = []
-    successful = 0
+    successful = len(already_downloaded)
 
     async with aiohttp.ClientSession() as session:
         # Process segments in batches

utils/times.py

@@ -3,6 +3,7 @@
 import asyncio
 import datetime
 import time
+from zoneinfo import ZoneInfo
 
 import aiohttp
 from tqdm import tqdm
@@ -10,6 +11,8 @@ from tqdm import tqdm
 from utils.stream import fetch_segment
 from utils.logging_config import logger
 
+FRANCE_TZ = ZoneInfo("Europe/Paris")
+
 
 def convert_ticks_to_sec(ticks, timescale):
     """Convert ticks to seconds."""
@@ -21,24 +24,24 @@ def convert_sec_to_ticks(seconds, timescale):
     return seconds * timescale
 
 
-def convert_sec_to_date(seconds, offset_hours=1):
-    """Convert seconds to datetime with offset."""
-    dt = datetime.datetime.utcfromtimestamp(seconds) + datetime.timedelta(
-        hours=offset_hours
+def convert_sec_to_date(seconds):
+    """Convert UTC seconds to France local datetime."""
+    dt = datetime.datetime.fromtimestamp(seconds, tz=datetime.UTC).astimezone(
+        FRANCE_TZ
     )
-    return dt
+    return dt.replace(tzinfo=None)
 
 
-def convert_date_to_sec(dt, offset_hours=1):
-    """Convert datetime to seconds with offset."""
-    epoch = datetime.datetime(1970, 1, 1)
-    utc_dt = dt - datetime.timedelta(hours=offset_hours)
-    return (utc_dt - epoch).total_seconds()
+def convert_date_to_sec(dt):
+    """Convert France local datetime to UTC seconds."""
+    aware = dt.replace(tzinfo=FRANCE_TZ)
+    epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.UTC)
+    return (aware.astimezone(datetime.UTC) - epoch).total_seconds()
 
 
-def convert_date_to_ticks(dt, timescale, offset_hours=1):
-    """Convert datetime to ticks with offset."""
-    return int(round(convert_date_to_sec(dt, offset_hours) * timescale))
+def convert_date_to_ticks(dt, timescale):
+    """Convert France local datetime to ticks."""
+    return int(round(convert_date_to_sec(dt) * timescale))
 
 
 def past(rep, base, duration):
@@ -99,9 +102,9 @@ async def bruteforce(track_id, date, batch_size=20000):
     return valid_ticks
 
 
-def find_nearest_tick_by_hour(base_tick, dt, timescale, duration, offset_hours=1):
+def find_nearest_tick_by_hour(base_tick, dt, timescale, duration):
     """Find the nearest tick for a given datetime."""
-    target_ticks = convert_date_to_ticks(dt, timescale, offset_hours)
+    target_ticks = convert_date_to_ticks(dt, timescale)
     diff_ticks = base_tick - target_ticks
     rep_estimate = diff_ticks / duration
 

utils/utilities.py

@@ -43,11 +43,11 @@ def merge_segments(input_folder: str, track_id: str, output_file: str):
     logger.info("Merged segments into %s", output_file)
 
 
-def decrypt(input_file, init_path, output_file, key):
-    """Decrypt a media file using mp4ff-decrypt.
+def decrypt(segment_dir, init_path, output_file, key):
+    """Decrypt segments in chunks of ~1GB to avoid loading entire file in memory.
 
     Args:
-        input_file: Path to the input encrypted file.
+        segment_dir: Path to the directory containing .m4s segment files.
         init_path: Path to the initialization file.
         output_file: Path to the output decrypted file.
         key: The decryption key in KID:KEY format.
@@ -56,14 +56,90 @@ def decrypt(input_file, init_path, output_file, key):
         True if decryption succeeded, False otherwise.
     """
     key = key.split(":")[1]
-    result = subprocess.run(
-        ["mp4ff-decrypt", "-init", init_path, "-key", key, input_file, output_file],
-        capture_output=True,
-        text=True,
-        check=False,
+
+    segment_files = sorted(
+        [f for f in os.listdir(segment_dir) if f.endswith(".m4s")],
+        key=lambda x: int(x.split(".")[0]),
     )
-    if result.returncode == 0:
-        logger.info("Decrypted %s to %s", input_file, output_file)
-        return True
-    logger.error("Decryption failed: %s", result.stderr)
-    return False
+
+    if not segment_files:
+        logger.error("No segment files found in %s", segment_dir)
+        return False
+
+    logger.info(
+        "Decrypting %d segments from %s to %s",
+        len(segment_files),
+        segment_dir,
+        output_file,
+    )
+
+    chunk_num = 0
+    temp_files = []
+    TARGET_SIZE = 1 * 1024 * 1024 * 1024 # 1GB
+
+    i = 0
+    while i < len(segment_files):
+        chunk_files = []
+        chunk_size = 0
+
+        while i < len(segment_files) and chunk_size < TARGET_SIZE:
+            fname = segment_files[i]
+            fpath = os.path.join(segment_dir, fname)
+            fsize = os.path.getsize(fpath)
+            chunk_files.append(fname)
+            chunk_size += fsize
+            i += 1
+
+        logger.debug(
+            "Processing chunk %d: %d segments, %.2f MB",
+            chunk_num,
+            len(chunk_files),
+            chunk_size / (1024 * 1024),
+        )
+
+        chunk_merged = os.path.join(segment_dir, f"chunk_{chunk_num}_merged")
+        chunk_dec = os.path.join(segment_dir, f"chunk_{chunk_num}_dec")
+        temp_files.extend([chunk_merged, chunk_dec])
+
+        with open(chunk_merged, "wb") as outfile:
+            for fname in chunk_files:
+                with open(os.path.join(segment_dir, fname), "rb") as infile:
+                    outfile.write(infile.read())
+
+        logger.debug("Decrypting chunk %d", chunk_num)
+        result = subprocess.run(
+            [
+                "mp4ff-decrypt",
+                "-init",
+                init_path,
+                "-key",
+                key,
+                chunk_merged,
+                chunk_dec,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0:
+            logger.error("Decryption failed for chunk %d: %s", chunk_num, result.stderr)
+            for tf in temp_files:
+                if os.path.exists(tf):
+                    os.remove(tf)
+            return False
+
+        chunk_num += 1
+
+    logger.debug("Concatenating %d decrypted chunks into %s", chunk_num, output_file)
+    with open(output_file, "wb") as outfile:
+        for c in range(chunk_num):
+            chunk_dec = os.path.join(segment_dir, f"chunk_{c}_dec")
+            with open(chunk_dec, "rb") as infile:
+                outfile.write(infile.read())
+
+    for tf in temp_files:
+        if os.path.exists(tf):
+            os.remove(tf)
+
+    logger.info("Successfully decrypted to %s", output_file)
+    return True