get_cartridges.py (10764B)
1 #!/usr/bin/env python3 2 """Télécharge le pack TOSEC EXL 100 (archive.org) et en extrait les cartouches [ROM] 3 dans roms/cartridges/<slug>.bin, avec roms/cartridges/titles.json (slug.bin -> titre). 4 5 Ces cartouches sont du logiciel Exelvision (société dissoute), préservé par TOSEC. Elles ne 6 sont PAS versionnées (roms/cartridges/ est dans .gitignore) ; la page les embarque dans dsp.data 7 et les exécute sans offrir de téléchargement par fichier. Lancé par « make cartridges ». 8 9 Usage : python3 -I tools/get_cartridges.py 10 """ 11 import glob 12 import io 13 import json 14 import os 15 import re 16 import subprocess 17 import sys 18 import tempfile 19 import urllib.request 20 import zipfile 21 import zlib 22 23 TOSEC_URL = ("https://ia600708.us.archive.org/6/items/pack-roms-exelvision-exl-100/" 24 "Exelvision%20EXL%20100%20%5BTOSEC%5D.rar") 25 HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 26 OUT = os.path.join(HERE, "roms", "cartridges") 27 28 # Titre, éditeur et année depuis un nom TOSEC « Tennis (198x)(Exelvision)(FR) ». 29 # Le 1er groupe (...) est l'année, le 2e l'éditeur, le reste le pays/flags. 30 def title_of(name): 31 base = os.path.splitext(os.path.basename(name))[0] 32 base = re.split(r"\s*\(", base, maxsplit=1)[0].strip() 33 return base or os.path.splitext(os.path.basename(name))[0] 34 35 def meta_of(name): 36 base = os.path.splitext(os.path.basename(name))[0] 37 title = title_of(name) 38 year, by = "", "" 39 for g in re.findall(r"\(([^)]*)\)", base): 40 g = g.strip() 41 if not g or g == "-": 42 continue 43 if not year and re.match(r"(19|20)[0-9x]{2}", g): 44 year = g 45 elif not by and g.lower() not in ("fr", "us", "eu", "en", "de", "es") and not re.fullmatch(r"[A-Za-z]{2}", g): 46 by = g 47 return title, by, year 48 49 def slug_of(title): 50 s = title.lower().replace("+", "-plus").replace("é", "e").replace("è", "e") 51 s = re.sub(r"[^a-z0-9]+", "-", s).strip("-") 52 return s or "cart" 53 54 # Catégorie depuis le dossier TOSEC (Games / Applications / Educational / Demos). 55 def cat_of(root): 56 if "Games" in root: 57 return "jeu" 58 if "Educational" in root: 59 return "educatif" 60 if "Applications" in root: 61 return "application" 62 if "Demos" in root: 63 return "demo" 64 return "" 65 66 # Cartouches présentes sur webmo.fr mais absentes du pack TOSEC (dumps bourrés à 32768, qui se 67 # chargent quand même). PINDO correspond au hash MAME. 68 WEBMO = "https://p7398.webmo.fr/httpdocs/exelvision/website/telechargement" 69 WEBMO_CARTS = { 70 "pindo.bin": ("PINDO.ROM", {"t": "Pindo", "by": "Minipuce", "y": "1986", "cat": "jeu"}), 71 "edigraf.bin": ("EDIGRAF.ROM", {"t": "Edigraf", "by": "P. Glajean", "y": "1987", "cat": "application"}), 72 "exelvox.bin": ("EXELVOX.ROM", {"t": "Exelvox", "by": "Exelvision", "y": "198x", "cat": "application"}), 73 } 74 WAV2K7 = os.path.join(HERE, "tools", "wav2k7.js") 75 MEDIAS_PAGE = "https://p7398.webmo.fr/httpdocs/exelvision/website/index.php?page=restauration-de-medias2" 76 77 def norm(s): 78 return re.sub(r"[^a-z0-9]", "", s.lower()) 79 80 # Cassettes présentes sur webmo.fr mais absentes de TOSEC : elles n'existent qu'en gros .wav, qu'on 81 # convertit en .k7 compact avec le décodeur de dsp (tools/wav2k7.js). Évite les doublons (nom normalisé). 82 def webmo_cassettes(titles): 83 have = {norm(m["t"]) for m in titles.values()} 84 page = urllib.request.urlopen( 85 urllib.request.Request(MEDIAS_PAGE, headers={"User-Agent": "Mozilla/5.0"}), timeout=60 86 ).read().decode("utf-8", "replace") 87 rars = sorted(set(re.findall(r"telechargement/([A-Za-z0-9_-]+)\.rar", page))) 88 added = 0 89 # Quelques doublons dont le nom webmo diffère trop du nom TOSEC pour être détecté autrement. 90 alias = {"ww3": "thirdworldwar", "symtrans": "symetriestranslations", 91 "faitesparlervotreexl100v1": "faitesparlervotreexl100volume1v11", 92 "faitesparlervotreexl100v2": "faitesparlervotreexl100volume2v11"} 93 for base in rars: 94 nb = norm(base) 95 if nb in have or re.sub(r"\d+$", "", nb) in have or alias.get(nb, nb) in have: 96 continue 97 # sous-chaîne (frise⊂frises, poker⊂exelpoker, vega⊂vegax4), noms assez longs 98 if any(len(nb) >= 4 and len(h) >= 4 and (nb in h or h in nb) for h in have): 99 continue 100 try: 101 data = urllib.request.urlopen(urllib.request.Request( 102 f"{WEBMO}/{base}.rar", headers={"User-Agent": "Mozilla/5.0"}), timeout=90).read() 103 except Exception as e: 104 print(f" {base}: téléchargement KO ({e})", file=sys.stderr) 105 continue 106 with tempfile.TemporaryDirectory() as td: 107 rp = os.path.join(td, base + ".rar") 108 with open(rp, "wb") as f: 109 f.write(data) 110 subprocess.run(["unar", "-q", "-D", "-o", td, rp], check=False, 111 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 112 wavs = sorted(glob.glob(os.path.join(td, "**", "*.wav"), recursive=True), 113 key=os.path.getsize, reverse=True) 114 if not wavs: 115 print(f" {base}: pas de .wav", file=sys.stderr) 116 continue 117 out = f"{slug_of(base)}.k7" 118 k7 = os.path.join(TAPES_OUT, out) 119 r = subprocess.run(["node", WAV2K7, wavs[0], k7], stderr=subprocess.DEVNULL) 120 if r.returncode != 0 or not os.path.exists(k7) or os.path.getsize(k7) < 16: 121 print(f" {base}: conversion KO", file=sys.stderr) 122 continue 123 titles[out] = {"t": base.replace("-", " ").title(), "by": "webmo.fr", "y": "", "cat": ""} 124 have.add(nb) 125 added += 1 126 print(f" {out:26s} {os.path.getsize(k7):6d} o (webmo) {base}", file=sys.stderr) 127 print(f"{added} cassettes webmo ajoutées", file=sys.stderr) 128 129 def webmo_cartridges(titles): 130 for out, (fn, meta) in WEBMO_CARTS.items(): 131 req = urllib.request.Request(f"{WEBMO}/{fn}", headers={"User-Agent": "Mozilla/5.0"}) 132 with urllib.request.urlopen(req, timeout=60) as r: 133 data = r.read() 134 with open(os.path.join(OUT, out), "wb") as f: 135 f.write(data) 136 titles[out] = meta 137 print(f" {out:22s} {len(data):6d} o (webmo.fr) {meta['t']}", file=sys.stderr) 138 139 def main(): 140 rar = os.path.join(tempfile.gettempdir(), "exl100-tosec.rar") 141 if not os.path.exists(rar) or os.path.getsize(rar) < 1_000_000: 142 print(f"Téléchargement {TOSEC_URL}", file=sys.stderr) 143 req = urllib.request.Request(TOSEC_URL, headers={"User-Agent": "Mozilla/5.0"}) 144 with urllib.request.urlopen(req, timeout=120) as r, open(rar, "wb") as f: 145 f.write(r.read()) 146 # Extraction complète puis filtrage des seuls dossiers de cartouches « - [ROM] ». 147 with tempfile.TemporaryDirectory() as tmp: 148 subprocess.run(["7z", "x", "-y", f"-o{tmp}", rar], check=True, stdout=subprocess.DEVNULL) 149 os.makedirs(OUT, exist_ok=True) 150 titles = {} 151 for root, _, files in os.walk(tmp): 152 if "- [ROM] " not in root: 153 continue 154 for fn in sorted(files): 155 if not fn.lower().endswith(".zip"): 156 continue 157 with zipfile.ZipFile(os.path.join(root, fn)) as z: 158 members = [m for m in z.namelist() if m.lower().endswith((".bin", ".rom"))] 159 if not members: 160 continue 161 data = z.read(members[0]) 162 title, by, year = meta_of(fn) 163 slug = slug_of(title) 164 out = f"{slug}.bin" 165 with open(os.path.join(OUT, out), "wb") as f: 166 f.write(data) 167 titles[out] = {"t": title, "by": by, "y": year, "cat": cat_of(root)} 168 print(f" {out:22s} {len(data):6d} o crc={zlib.crc32(data) & 0xffffffff:08x} {title} — {by} {year}", 169 file=sys.stderr) 170 webmo_cartridges(titles) # PINDO, EDIGRAF, EXELVOX (absents de TOSEC) 171 with open(os.path.join(OUT, "titles.json"), "w", encoding="utf-8") as f: 172 json.dump(dict(sorted(titles.items(), key=lambda kv: kv[1]["t"].lower())), f, 173 ensure_ascii=False, indent=1) 174 print(f"{len(titles)} cartouches dans {OUT}", file=sys.stderr) 175 extract_tapes(tmp) 176 177 # Cassettes [K7] (programmes BASIC) : jeux et applications, versions de base seulement. 178 TAPES_OUT = os.path.join(HERE, "roms", "tapes") 179 TAPE_SKIP = ("[h ", "[a]", "[b", "[req 64kb]", "side b", "(beta)", "[unreleased]", "(side a)") 180 def extract_tapes(tmp, limit=160): 181 os.makedirs(TAPES_OUT, exist_ok=True) 182 titles = {} 183 for root, _, files in os.walk(tmp): 184 if "- [K7] " not in root: 185 continue 186 # Jeux, applications, éducatif ET démos (dont « Faites parler votre EXL100 »). 187 if not any(c in root for c in ("Games", "Educational", "Applications", "Demos")): 188 continue 189 for fn in sorted(files): 190 if not fn.lower().endswith(".zip"): 191 continue 192 low = fn.lower() 193 if any(s in low for s in TAPE_SKIP): 194 continue 195 with zipfile.ZipFile(os.path.join(root, fn)) as z: 196 members = [m for m in z.namelist() if m.lower().endswith((".k7", ".wav"))] 197 if not members: 198 continue 199 data = z.read(members[0]) 200 if not data: 201 continue 202 title, by, year = meta_of(fn) 203 slug = slug_of(title) 204 out = f"{slug}.k7" 205 if out in titles: # doublon de titre : on garde le premier 206 continue 207 with open(os.path.join(TAPES_OUT, out), "wb") as f: 208 f.write(data) 209 titles[out] = {"t": title, "by": by, "y": year, "cat": cat_of(root)} 210 if len(titles) >= limit: 211 break 212 if len(titles) >= limit: 213 break 214 # NOTE : webmo_cassettes(titles) ajouterait les cassettes exclusives à webmo.fr, mais elles n'y 215 # existent qu'en enregistrements .wav analogiques : converties en .k7 (tools/wav2k7.js), elles se 216 # MONTENT et démarrent le chargement (ESC -> Lect.) mais les données ne se chargent pas en programme 217 # fiable (décodage avec pertes). On s'en tient donc au corpus TOSEC, propre et chargeable. 218 with open(os.path.join(TAPES_OUT, "titles.json"), "w", encoding="utf-8") as f: 219 json.dump(dict(sorted(titles.items(), key=lambda kv: kv[1]["t"].lower())), f, 220 ensure_ascii=False, indent=1) 221 print(f"{len(titles)} cassettes dans {TAPES_OUT}", file=sys.stderr) 222 223 if __name__ == "__main__": 224 main()