noham/OqeeRewind
1
0
Fork 0
mirror of https://github.com/NohamR/OqeeRewind.git synced 2026-08-26 02:29:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

optimize DASH probing and segment download paths

Browse Source 
- Switch bruteforce probes from GET to HEAD to avoid downloading
  full segment bodies (only the status code is needed)
- Cap bruteforce concurrency with a semaphore (default 100) instead of issuing one task per tick, and cancel leftover tasks on early exit
- Reuse a single aiohttp session / event loop for the manifest mode's AVC/HEVC/audio bruteforces to keep TCP connections alive
- Hoist the per-request segment headers into a module-level constant
- Use a set for the already-downloaded resume check in save_segments
- Replace read()-into-memory copies with shutil.copyfileobj in   merge_segments and decrypt
This commit is contained in:
√(noham)²
2026-08-13 17:40:06 +02:00
parent a57f0a74a1
commit ae5035c6a8
4 changed files with 114 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
utils/stream.py
View File

@@ -18,6 +18,16 @@ ET.register_namespace("", "urn:mpeg:dash:schema:mpd:2011")
ET.register_namespace("xsi", "http://www.w3.org/2001/XMLSchema-instance")
ET.register_namespace("cenc", "urn:mpeg:cenc:2013")
SEGMENT_HEADERS = {
"Accept": "*/*",
"Referer": "https://tv.free.fr/",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/143.0.0.0 Safari/537.36"
),
}
def parse_mpd_manifest(mpd_content: str) -> Dict[str, Any]:
"""Parse an MPD manifest and extract metadata.
@@ -340,7 +350,7 @@ def get_manifest(manifest_id):
async def fetch_segment(session, ticks, track_id):
"""Fetch a media segment asynchronously.
"""Probe whether a media segment exists using a lightweight HEAD request.
Args:
session: The aiohttp ClientSession.
@@ -348,20 +358,11 @@ async def fetch_segment(session, ticks, track_id):
track_id: The track identifier.
Returns:
The tick value if successful, None otherwise.
The tick value if the segment exists, None otherwise.
"""
url = f"https://media4.stream.proxad.net/media/{track_id}_{ticks}"
headers = {
"Accept": "*/*",
"Referer": "https://tv.free.fr/",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/143.0.0.0 Safari/537.36"
),
}
try:
async with session.get(url, headers=headers) as resp:
async with session.head(url, headers=SEGMENT_HEADERS) as resp:
if resp.status == 200:
return ticks
return None
@@ -377,16 +378,7 @@ def get_init(output_folder, track_id):
track_id: The track identifier.
"""
url = f"https://media4.stream.proxad.net/media/{track_id}_init"
headers = {
"Accept": "*/*",
"Referer": "https://tv.free.fr/",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/143.0.0.0 Safari/537.36"
),
}
response = requests.get(url, headers=headers, timeout=10)
response = requests.get(url, headers=SEGMENT_HEADERS, timeout=10)
if response.status_code == 200:
os.makedirs(f"{output_folder}/segments_{track_id}", exist_ok=True)
init_path = f"{output_folder}/segments_{track_id}/init.mp4"
@@ -418,17 +410,8 @@ async def save_segments(
Tuple of (success: bool, tick: int, rep: int)
"""
url = f"https://media4.stream.proxad.net/media/{track_id}_{tick}"
headers = {
"Accept": "*/*",
"Referer": "https://tv.free.fr/",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/143.0.0.0 Safari/537.36"
),
}
try:
async with session.get(url, headers=headers) as resp:
async with session.get(url, headers=SEGMENT_HEADERS) as resp:
if resp.status == 200:
content = await resp.read()
filename = f"{output_folder}/segments_{track_id}/{tick}.m4s"
@@ -458,11 +441,11 @@ async def save_segments(
segments_to_download = [(start_tick + i * duration, i) for i in range(rep_nb)]
# In case of resuming, check which segments are already downloaded
already_downloaded = [
already_downloaded = {
int(f.split(".")[0])
for f in os.listdir(f"{output_folder}/segments_{track_id}")
if f.endswith(".m4s") and f.split(".")[0].isdigit()
]
}
segments_to_download = [
(tick, rep) for tick, rep in segments_to_download if tick not in already_downloaded
]

71
utils/times.py
View File

@@ -55,45 +55,66 @@ def future(rep, base, duration):
return base + rep * duration
async def bruteforce(track_id, date, batch_size=20000):
"""Bruteforce segments to find valid ticks."""
async def bruteforce(track_id, date, batch_size=20000, concurrency=100, session=None):
"""Bruteforce segments to find valid ticks.
Args:
track_id: The track identifier.
date: The base tick to start probing from.
batch_size: Number of ticks to probe per batch.
concurrency: Maximum number of in-flight requests at once.
session: Optional shared aiohttp session. A new one is created if omitted.
Returns:
List of valid ticks found (empty if none).
"""
valid_ticks = []
total_requests = 288000
logger.debug("Starting bruteforce for %s near (%s) %s", track_id, date, convert_sec_to_date(convert_ticks_to_sec(date, TIMESCALE)))
start_time = time.time()
owns_session = session is None
if owns_session:
session = aiohttp.ClientSession()
try:
async with aiohttp.ClientSession() as session:
for batch_start in range(0, total_requests, batch_size):
batch_end = min(batch_start + batch_size, total_requests)
tasks = [
fetch_segment(session, t + date, track_id)
for t in range(batch_start, batch_end)
]
semaphore = asyncio.Semaphore(concurrency)
results = []
for coro in tqdm(
asyncio.as_completed(tasks),
total=len(tasks),
desc="Bruteforce",
unit="req",
):
result = await coro
results.append(result)
async def probe(t):
async with semaphore:
return await fetch_segment(session, t + date, track_id)
valid_ticks.extend(
[r for r in results if r and not isinstance(r, Exception)]
)
for batch_start in range(0, total_requests, batch_size):
batch_end = min(batch_start + batch_size, total_requests)
tasks = [probe(t) for t in range(batch_start, batch_end)]
# Stop if we found valid ticks
if valid_ticks:
logger.debug("Found valid ticks: %s, stopping bruteforce.", valid_ticks)
break
results = []
for coro in tqdm(
asyncio.as_completed(tasks),
total=len(tasks),
desc="Bruteforce",
unit="req",
):
result = await coro
results.append(result)
valid_ticks.extend(
[r for r in results if r and not isinstance(r, Exception)]
)
# Stop if we found valid ticks
if valid_ticks:
logger.debug("Found valid ticks: %s, stopping bruteforce.", valid_ticks)
for task in tasks:
task.cancel()
break
except KeyboardInterrupt:
logger.error("Interrupted by user (Ctrl+C)")
finally:
if owns_session:
await session.close()
elapsed = time.time() - start_time
logger.debug("Completed in %.2fs", elapsed)

4
utils/utilities.py
View File

@@ -39,7 +39,7 @@ def merge_segments(input_folder: str, track_id: str, output_file: str):
with open(output_file, "wb") as outfile:
for fname in segment_files:
with open(f"{segment_folder}/{fname}", "rb") as infile:
outfile.write(infile.read())
shutil.copyfileobj(infile, outfile)
logger.info("Merged segments into %s", output_file)
@@ -104,7 +104,7 @@ def decrypt(segment_dir, init_path, output_file, key):
with open(chunk_merged, "wb") as outfile:
for fname in chunk_files:
with open(os.path.join(segment_dir, fname), "rb") as infile:
outfile.write(infile.read())
shutil.copyfileobj(infile, outfile)
logger.debug("Decrypting chunk %d", chunk_num)
result = subprocess.run(