208 lines
14 KiB
Python
208 lines
14 KiB
Python
import os
|
|
import io
|
|
import json
|
|
import re
|
|
import pandas as pd
|
|
import pdfplumber
|
|
import unicodedata
|
|
import traceback
|
|
from flask import Flask, request, render_template_string, Response, send_from_directory, redirect
|
|
|
|
app = Flask(__name__)
|
|
|
|
# Dossiers et fichiers
|
|
TEMP_DIR = "/app/downloads"
|
|
if not os.path.exists(TEMP_DIR): os.makedirs(TEMP_DIR)
|
|
|
|
TEMP_DATA_FILE = os.path.join(TEMP_DIR, "temp_data.json")
|
|
TEMP_MISSING_FILE = os.path.join(TEMP_DIR, "temp_missing.json")
|
|
CSV_FILE = "/app/corresp_IGPDE.csv"
|
|
ODS_FILE = "/app/corresp_IGPDE.ods"
|
|
RESULT_FILE = os.path.join(TEMP_DIR, "resultat.xlsx")
|
|
|
|
# --- GESTION DE LA BASE DE DONNÉES ---
|
|
def ensure_database():
|
|
if os.path.exists(CSV_FILE): return
|
|
if os.path.exists(ODS_FILE):
|
|
try:
|
|
df = pd.read_excel(ODS_FILE, engine="odf")
|
|
df.to_csv(CSV_FILE, index=False, encoding='utf-8-sig')
|
|
return
|
|
except: pass
|
|
df_empty = pd.DataFrame(columns=["Nouvelle nomenclature", "Nomenclature IGPDE"])
|
|
df_empty.to_csv(CSV_FILE, index=False, encoding='utf-8-sig')
|
|
|
|
# --- FONCTION DE NORMALISATION ---
|
|
def super_norm(texte):
|
|
if not texte or not isinstance(texte, str): return ""
|
|
t = "".join(c for c in unicodedata.normalize('NFD', texte) if unicodedata.category(c) != 'Mn')
|
|
t = re.sub(r'[^a-zA-Z0-9]', '', t)
|
|
return t.lower()
|
|
|
|
# --- DESIGN CSS ---
|
|
COMMON_STYLE = """
|
|
<style>
|
|
body { font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; background: #f4f7f9; color: #333; margin: 0; padding: 20px; }
|
|
.nav-bar { display: flex; gap: 10px; margin-bottom: 20px; justify-content: center; }
|
|
.nav-btn { background: #fff; border: 1px solid #cbd5e0; padding: 8px 15px; border-radius: 8px; text-decoration: none; color: #4a5568; font-size: 0.9rem; transition: 0.2s; }
|
|
.nav-btn:hover { background: #edf2f7; }
|
|
.nav-btn.active { background: #3182ce; color: white; border-color: #3182ce; }
|
|
.card { background: white; padding: 2rem; border-radius: 15px; box-shadow: 0 4px 15px rgba(0,0,0,0.05); max-width: 1200px; margin: 0 auto; }
|
|
h1 { color: #2d3748; margin-top: 0; font-size: 1.5rem; }
|
|
.header-box { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; }
|
|
.btn-submit { background: #3182ce; color: white; border: none; padding: 10px 20px; border-radius: 8px; font-size: 0.9rem; cursor: pointer; font-weight: 600; text-decoration: none; display: inline-block; }
|
|
.btn-submit:hover { background: #2b6cb0; }
|
|
#log-window { background: #1a202c; color: #cbd5e0; padding: 15px; border-radius: 10px; height: 350px; overflow-y: auto; font-family: 'Courier New', monospace; font-size: 0.85rem; text-align: left; margin-top: 20px; line-height: 1.6; border: 1px solid #2d3748; }
|
|
.log-success { color: #68d391; font-weight: bold; }
|
|
.log-warn { color: #fc8181; }
|
|
.table-container { overflow-x: auto; border-radius: 10px; border: 1px solid #e2e8f0; max-height: 600px; }
|
|
table { width: 100%; border-collapse: collapse; background: white; font-size: 0.85rem; }
|
|
th { background: #f8fafc; padding: 10px; border-bottom: 2px solid #e2e8f0; text-align: left; position: sticky; top: 0; }
|
|
td { padding: 8px 10px; border-bottom: 1px solid #edf2f7; }
|
|
</style>
|
|
"""
|
|
|
|
def get_nav(active_page):
|
|
h = "active" if active_page=="home" else ""
|
|
c = "active" if active_page=="corresp" else ""
|
|
r = "active" if active_page=="result" else ""
|
|
return f'<div class="nav-bar"><a href="/" class="nav-btn {h}">🏠 Accueil</a><a href="/view_correspondence" class="nav-btn {c}">📋 Table de Correspondance</a><a href="/view_result" class="nav-btn {r}">📊 Dernier Résultat</a></div>'
|
|
|
|
@app.route("/")
|
|
def home():
|
|
ensure_database()
|
|
return render_template_string(f'<!DOCTYPE html><html><head><meta charset="UTF-8">{COMMON_STYLE}</head><body>{get_nav("home")}<div class="card" style="text-align:center; max-width: 500px;"><div style="font-size: 3rem; margin-bottom: 1rem;">📄</div><h1>Extraction PDF</h1><form action="/process" method="post" enctype="multipart/form-data"><input type="file" name="pdf_file" accept=".pdf" required id="file-input" style="display:none;" onchange="document.getElementById(\'file-label\').innerText = this.files[0].name"><label for="file-input" id="file-label" style="display:block; border: 2px dashed #cbd5e0; padding: 2rem; border-radius: 10px; cursor: pointer; margin-bottom:1.5rem; background:#f8fafc;">📂 Choisir le PDF</label><button type="submit" class="btn-submit">Lancer l\'analyse</button></form></div></body></html>')
|
|
|
|
@app.route("/view_correspondence")
|
|
def view_correspondence():
|
|
ensure_database()
|
|
df = pd.read_csv(CSV_FILE, encoding='utf-8-sig').fillna('')
|
|
table_html = df.to_html(index=False)
|
|
return render_template_string(f'<!DOCTYPE html><html><head>{COMMON_STYLE}</head><body>{get_nav("corresp")}<div class="card"><div class="header-box"><h1>Table de Correspondance (CSV)</h1><a href="/download_db" class="btn-submit">📥 Télécharger la base (CSV)</a></div><div class="table-container">{table_html}</div></div></body></html>')
|
|
|
|
@app.route("/view_result")
|
|
def view_result():
|
|
if not os.path.exists(RESULT_FILE): return "Aucun résultat."
|
|
df = pd.read_excel(RESULT_FILE).fillna('')
|
|
table_html = df.to_html(index=False)
|
|
return render_template_string(f'<!DOCTYPE html><html><head>{COMMON_STYLE}</head><body>{get_nav("result")}<div class="card"><div class="header-box"><h1>Dernier Résultat Extrait</h1><a href="/download" class="btn-submit">📥 Télécharger le résultat (Excel)</a></div><div class="table-container">{table_html}</div></div></body></html>')
|
|
|
|
@app.route("/process", methods=["POST"])
|
|
def process():
|
|
ensure_database()
|
|
file = request.files['pdf_file']
|
|
if not file: return "Fichier manquant"
|
|
file_content = file.read()
|
|
|
|
def generate():
|
|
yield f'<!DOCTYPE html><html lang="fr"><head><meta charset="UTF-8">{COMMON_STYLE}</head><body><div class="card"><h1>Analyse en cours...</h1><div style="background:#e2e8f0; height:12px; border-radius:6px; overflow:hidden;"><div id="prog" style="background:#3182ce; width:0%; height:100%; transition:0.3s;"></div></div><div id="log-window"></div></div><script>const logWin = document.getElementById("log-window"); const prog = document.getElementById("prog"); function addLog(msg, type="info", p=null) {{ const div = document.createElement("div"); div.className = "log-" + type; div.innerHTML = "> " + msg; logWin.appendChild(div); logWin.scrollTop = logWin.scrollHeight; if(p !== null) prog.style.width = p + "%"; }}</script>'
|
|
|
|
all_formations = []
|
|
try:
|
|
with pdfplumber.open(io.BytesIO(file_content)) as pdf:
|
|
total_pages = len(pdf.pages)
|
|
for i, page in enumerate(pdf.pages):
|
|
page_num = i + 1
|
|
percent = int((page_num / total_pages) * 85)
|
|
try:
|
|
check_zone = page.crop((page.width * 0.5, 0, page.width, 400))
|
|
if "Référence" in (check_zone.extract_text() or ""):
|
|
bandeaux = sorted([r for r in page.rects if (r['x1'] - r['x0']) > page.width * 0.7 and r['top'] < 300 and 15 < r['height'] < 200], key=lambda r: r['top'])
|
|
if len(bandeaux) >= 2:
|
|
titre_brut = " ".join((page.crop((0, max(0, bandeaux[0]['top'] - 2), page.width, min(page.height, bandeaux[0]['bottom'] + 2))).extract_text() or "").replace('\n', ' ').split())
|
|
categorie_str = " ".join((page.crop((0, max(0, bandeaux[1]['top'] - 2), page.width, min(page.height, bandeaux[1]['bottom'] + 2))).extract_text() or "").replace('\n', ' ').split())
|
|
else:
|
|
header = page.crop((0, 60, page.width, 250))
|
|
lines = [l.strip() for l in (header.extract_text() or "").split('\n') if l.strip() and not (len(l) > 90 or l.endswith('.'))]
|
|
titre_brut, categorie_str = (" ".join(lines[:-1]), lines[-1]) if len(lines) >= 2 else (lines[0] if lines else "", "")
|
|
|
|
chapitre, sous_chap = (categorie_str.split(">")[0].strip(), categorie_str.split(">")[1].strip()) if ">" in categorie_str else (categorie_str.strip(), "")
|
|
data = {'Nom_IGPDE_Key': f"{chapitre} > {sous_chap}" if sous_chap else chapitre, 'Chapitre': chapitre, 'Sous-chapitre': sous_chap, 'Titre': titre_brut, 'Page_Source': page_num}
|
|
|
|
table = page.crop((page.width * 0.6, 180, page.width, 600)).extract_table()
|
|
if table:
|
|
for row in table:
|
|
if len(row) >= 2 and row[0]:
|
|
cle = row[0].replace('\n', ' ').strip()
|
|
for col in ["Référence", "Durée", "Type", "Publics éligibles", "Niveau", "Sessions", "Tarifs"]:
|
|
if col.lower() in cle.lower():
|
|
data[col] = row[1].replace('\n', ' ').strip()
|
|
break
|
|
all_formations.append(data)
|
|
t_log = titre_brut[:40].replace("'", " ").replace('"', ' ')
|
|
yield f"<script>addLog('Page {page_num} : {t_log}...', 'success', {percent})</script>"
|
|
except Exception: pass
|
|
|
|
# --- MAPPING ---
|
|
mapping_dict = {}
|
|
df_map = pd.read_csv(CSV_FILE, encoding='utf-8-sig')
|
|
for _, row in df_map.dropna(subset=['Nomenclature IGPDE']).iterrows():
|
|
mapping_dict[super_norm(str(row['Nomenclature IGPDE']))] = str(row['Nouvelle nomenclature']).strip()
|
|
|
|
missing = []
|
|
seen_missing_norm = set()
|
|
for f in all_formations:
|
|
full_norm, chap_norm = super_norm(f['Nom_IGPDE_Key']), super_norm(f['Chapitre'])
|
|
if full_norm not in mapping_dict and chap_norm not in mapping_dict:
|
|
if full_norm not in seen_missing_norm:
|
|
missing.append(f['Nom_IGPDE_Key']); seen_missing_norm.add(full_norm)
|
|
|
|
with open(TEMP_DATA_FILE, "w", encoding="utf-8") as f: json.dump(all_formations, f)
|
|
if missing:
|
|
with open(TEMP_MISSING_FILE, "w", encoding="utf-8") as f: json.dump(missing, f)
|
|
yield "<script>setTimeout(() => window.location.href='/ask_mapping', 800);</script>"
|
|
else:
|
|
yield "<script>setTimeout(() => window.location.href='/finalize', 800);</script>"
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
yield f"<script>addLog('Erreur : {str(e)}', 'warn')</script>"
|
|
yield "</body></html>"
|
|
return Response(generate(), mimetype='text/html')
|
|
|
|
@app.route("/ask_mapping")
|
|
def ask_mapping():
|
|
with open(TEMP_MISSING_FILE, "r", encoding="utf-8") as f: missing = json.load(f)
|
|
df_db = pd.read_csv(CSV_FILE, encoding='utf-8-sig')
|
|
known = sorted(df_db['Nouvelle nomenclature'].dropna().unique().tolist())
|
|
options_html = "".join([f'<option value="{kn}">' for kn in known])
|
|
html = f'<!DOCTYPE html><html><head>{COMMON_STYLE}</head><body><div class="card"><h1>Nouveaux Chapitres</h1><form action="/save_mapping" method="POST">'
|
|
for i, cat in enumerate(missing):
|
|
html += f'<div style="margin-bottom:15px; background:#f7fafc; padding:15px; border-radius:10px;"><label style="display:block; font-weight:600; font-size:0.85rem; margin-bottom:5px;">{cat}</label><input type="hidden" name="igpde_{i}" value="{cat}"><input type="text" name="nouv_{i}" list="known_noms" placeholder="Associer à..." style="width:100%; padding:10px; border:1px solid #cbd5e0; border-radius:6px;"></div>'
|
|
html += f'<datalist id="known_noms">{options_html}</datalist><button type="submit" class="btn-submit" style="width:100%">Valider</button></form></div></body></html>'
|
|
return render_template_string(html)
|
|
|
|
@app.route("/save_mapping", methods=["POST"])
|
|
def save_mapping():
|
|
new_rows = []
|
|
for k in [k for k in request.form.keys() if k.startswith("igpde_")]:
|
|
idx = k.split("_")[1]
|
|
new_rows.append({"Nouvelle nomenclature": request.form.get(f"nouv_{idx}", "").strip(), "Nomenclature IGPDE": request.form.get(k)})
|
|
if new_rows:
|
|
df_final = pd.concat([pd.read_csv(CSV_FILE, encoding='utf-8-sig'), pd.DataFrame(new_rows)], ignore_index=True)
|
|
df_final.to_csv(CSV_FILE, index=False, encoding='utf-8-sig')
|
|
return redirect("/finalize")
|
|
|
|
@app.route("/finalize")
|
|
def finalize():
|
|
with open(TEMP_DATA_FILE, "r", encoding="utf-8") as f: all_formations = json.load(f)
|
|
mapping_dict = {}
|
|
df_map = pd.read_csv(CSV_FILE, encoding='utf-8-sig').dropna(subset=['Nomenclature IGPDE'])
|
|
for _, row in df_map.iterrows():
|
|
mapping_dict[super_norm(str(row['Nomenclature IGPDE']))] = str(row['Nouvelle nomenclature']).strip()
|
|
for f in all_formations:
|
|
key = f.pop('Nom_IGPDE_Key', '')
|
|
f['Nouvelle nomenclature'] = mapping_dict.get(super_norm(key), mapping_dict.get(super_norm(f['Chapitre']), ""))
|
|
df = pd.DataFrame(all_formations).fillna('')
|
|
cols = ['Nouvelle nomenclature', 'Chapitre', 'Sous-chapitre', 'Titre', 'Page_Source'] + [c for c in df.columns if c not in ['Nouvelle nomenclature', 'Chapitre', 'Sous-chapitre', 'Titre', 'Page_Source']]
|
|
df_f = df[[c for c in cols if c in df.columns]]
|
|
df_f.to_excel(RESULT_FILE, index=False, engine='xlsxwriter')
|
|
return redirect("/view_result")
|
|
|
|
@app.route("/download")
|
|
def download(): return send_from_directory(TEMP_DIR, "resultat.xlsx", as_attachment=True)
|
|
|
|
@app.route("/download_db")
|
|
def download_db(): return send_from_directory("/app", "corresp_IGPDE.csv", as_attachment=True)
|
|
|
|
if __name__ == "__main__": app.run(host='0.0.0.0', port=5000)
|