first crawl

Makefile

@@ -1,11 +1,10 @@
 TEX=pdflatex
-SRC=essaie.tex
+SRC=file.tex
 OUTDIR=output
-OUTPDF=$(OUTDIR)/essaie.pdf
 
 all:
 	@mkdir -p $(OUTDIR)
-	$(TEX) -output-directory=$(OUTDIR) $(SRC)
+	$(TEX) -output-directory=$(OUTDIR) $(SRC) -interaction=batchmode
 
 clean:
 	rm -rf $(OUTDIR)/*.aux $(OUTDIR)/*.log $(OUTDIR)/*.pdf

grab.py

@@ -1,8 +1,131 @@
+from pprint import pprint
 from bs4 import BeautifulSoup
 import requests
 import urllib.parse as urlparse
 
-headers = {
+class Exercise:
+    def __init__(self, number, title, id, stars, enonce, indication, answer):
+        self.number = number
+        self.title = title
+        self.id = id
+        self.stars = stars
+        self.enonce = enonce
+        self.indication = indication
+        self.answer = answer
+
+    def __repr__(self):
+        return f"Exercise({self.number}, {self.title}, {self.id}, {self.stars})"
+    
+class Part:
+    def __init__(self, title):
+        self.title = title
+        self.exercises = []
+
+    def add_exercise(self, exercise):
+        self.exercises.append(exercise)
+
+    def __repr__(self):
+        return f"Part({self.title}, {len(self.exercises)} exercises)"
+    
+class Chapitre:
+    def __init__(self, title, url=None):
+        self.title = title
+        self.parts = []
+        self.url = url
+
+    def add_part(self, part):
+        self.parts.append(part)
+
+    def __repr__(self):
+        return f"Chapitre({self.title}, {len(self.parts)} parts)"
+
+    def show(self):
+        print(f"Chapitre: {self.title}")
+        for part in self.parts:
+            print(f"  Part: {part.title}")
+            for exercise in part.exercises:
+                print(f"    Exercise: {exercise.number}, {exercise.title}, {exercise.id}, {exercise.stars} stars")
+                print(f"      Enonce: {exercise.enonce}")
+                print(f"      Indication: {exercise.indication}")
+                print(f"      Answer: {exercise.answer}")
+    
+class LatexFile:
+    def __init__(self, filename):
+        self.filename = filename
+        self.content = ""
+
+    def add_header(self):
+        with open('parts/header.tex', 'r') as f:
+            self.content += f.read()
+
+    def add_footer(self):
+        with open('parts/footer.tex', 'r') as f:
+            self.content += f.read()
+
+    def add_content(self, content):
+        self.content += content + "\n"
+
+    def add_pagebreak(self):
+        self.content += "\\newpage\n"
+
+    def add_source(self, chapitre):
+        self.content += f"\\noindent\\textbf{{Chapitre:}} \\href{{{chapitre.url}}}{{{chapitre.title}}}\n"
+
+    def add_exercise(self, exercise):
+        self.content += f"""\\exercice{{{exercise.number}, name, date, {exercise.stars}, {exercise.title}}}\n"""
+        self.content += f"""\\enonce{{{exercise.number}}}{{}}\n"""
+        self.content += f"{exercise.enonce}\n"
+        self.content += f"""\\finenonce{{{exercise.number}}}\n"""
+        self.content += f"""\\finexercice\n"""
+        self.content += "\n"
+
+    def add_indication(self, exercise):
+        self.content += f"""\\indication{{{exercise.number}}}\n"""
+        self.content += f"{exercise.indication}\n"
+        self.content += f"""\\finindication\n"""
+        self.content += "\n"
+
+    def add_answer(self, exercise):
+        self.content += f"""\\correction{{{exercise.number}}}\n"""
+        self.content += f"{exercise.answer}\n"
+        self.content += f"""\\fincorrection\n"""
+        self.content += "\n"
+
+    def sanitize(self):
+        # return ''.join(c for c in self.content if ord(c) < 128)
+        return self.content.encode('utf-8', 'ignore').decode('utf-8')
+
+    def save(self):
+        with open(self.filename, 'w') as f:
+            f.write(self.sanitize())
+
+    def generate_latex(self, chapitre):
+        self.add_header()
+
+        self.add_source(chapitre)
+
+        self.add_content(f"\\title{{{chapitre.title}}}")
+        for part in chapitre.parts:
+            self.add_content(f"\\section{{{part.title}}}")
+            for ex in part.exercises:
+                self.add_exercise(ex)
+        
+        self.add_pagebreak()
+
+        for part in chapitre.parts:
+            for ex in part.exercises:
+                self.add_indication(ex)
+        self.add_pagebreak()
+
+        for part in chapitre.parts:
+            for ex in part.exercises:
+                self.add_answer(ex)
+
+        self.add_footer()
+        self.save()
+
+def fetch_chapitre(quoi):
+    headers = {
         'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
         'accept-language': 'fr-FR,fr;q=0.7',
         'cache-control': 'no-cache',
@@ -18,32 +141,28 @@ headers = {
         'sec-gpc': '1',
         'upgrade-insecure-requests': '1',
         'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-}
+    }
-
+    params = {
-params = {
         'action': 'affiche',
-    'quoi': 'mpsi/feuillesexo/matrices',
+        'quoi': quoi,
         'type': 'fexo',
-}
+    }
-
+    response = requests.get('https://bibmath.net/ressources/index.php', headers=headers, params=params)
-response = requests.get('https://bibmath.net/ressources/index.php', headers=headers, params=params)
+    if response.status_code == 200:
-
-if response.status_code == 200:
         soup = BeautifulSoup(response.text, 'html.parser')
         article = soup.find('article', id='contenugauche')
+        
         if article:
             title = article.find('h1').get_text(strip=True)
-        print(f"Title: {title}")
+            chapitre = Chapitre(title, url=response.url)
-        
+            current_part = None
-        # Find all part titles
+            for element in article.find_all(recursive=False):
-        part_titles = article.find_all('div', class_='titrepartie')
+                if 'titrepartie' in element.get('class', []):
-        for part in part_titles:
+                    part_title = element.get_text(strip=True)
-            part_text = part.get_text(strip=True)
+                    current_part = Part(part_title)
-            print(f"Part of the File: {part_text}")
+                    chapitre.add_part(current_part)
-        
+                elif 'exo' in element.get('class', []) and current_part is not None:
-        exercises = article.find_all('div', class_='exo')
+                    exo_title_div = element.find('div', class_='titreexo')
-        for exo in exercises:
-            exo_title_div = exo.find('div', class_='titreexo')
 
                     exercise_number = exo_title_div.find_all('span')[1].text.strip()
                     title_start = exo_title_div.text.find(' - ') + 3
@@ -56,26 +175,39 @@ if response.status_code == 200:
 
                     stars_count = len(exo_title_div.find_all('img'))
 
-            enonce = exo.find('div', class_='enonce').find('div', class_='inner').get_text(strip=True)
+                    enonce = element.find('div', class_='enonce').find('div', class_='inner').get_text(strip=True)
-            indication_div = exo.find('div', class_='indication')
+                    indication_div = element.find('div', class_='indication')
                     indication = (
                         indication_div.find('div', class_='inner').get_text(strip=True) 
                         if indication_div else "No indication"
                     )
 
-            answer_div = exo.find('div', class_='corrige')
+                    answer_div = element.find('div', class_='corrige')
                     answer = (
                         answer_div.find('div', class_='inner').get_text(strip=True)
                         if answer_div else "No answer"
                     )
-            # print(f"Exercise Number: {exercise_number}")
+
-            # print(f"Exercise Title: {exercise_title}")
+                    exercise = Exercise(
-            # print(f"Exercise ID: {exercise_id}")
+                        number=exercise_number,
-            # print(f"Stars: {stars_count} stars")
+                        title=exercise_title,
-            # print(f"Enonce: {enonce}")
+                        id=exercise_id,
-            # print(f"Indication: {indication}")
+                        stars=stars_count,
-            # print(f"Answer: {answer}")
+                        enonce=enonce,
+                        indication=indication,
+                        answer=answer
+                    )
+                    
+                    current_part.add_exercise(exercise)
+            return chapitre
         else:
             print("Article with id 'contenugauche' not found.")
-else:
+    else:
         print(f"Request failed with status code: {response.status_code}")
+
+quoi = 'mpsi/feuillesexo/matrices'
+chapitre = fetch_chapitre(quoi)
+
+latex_file = LatexFile('file.tex')
+latex_file.generate_latex(chapitre)
+print("LaTeX file generated: file.tex")

parts/header.tex

@@ -106,5 +106,5 @@
 \usepackage{graphics}
 \usepackage[all]{xy}
 
-
+\pagestyle{empty}
 \begin{document}