野獣大百科のコメント一覧アーカイブ化スクリプト

@korosu_gmnt / 更新: 2026/01/10 00:29

HTMLとdatを生成します

TEXT 5.77KB
1
  1. from bs4 import BeautifulSoup
  2. import html, requests, time, os, json
  4. ws_dir = "./yajudic"
  5. os.makedirs(ws_dir, exist_ok=True)
  7. proxies = {
  8. "http": "socks5h://127.0.0.1:9150",
  9. "https": "socks5h://127.0.0.1:9150",
  10. }
  12. def normalize_comment(node):
  13. parts = []
  14. for s in node.stripped_strings:
  15. parts.append(s)
  16. return "\n".join(parts)
  18. def save_json(wid):
  19. output_dir = os.path.join(ws_dir, f"{wid}")
  20. os.makedirs(output_dir, exist_ok=True)
  22. page = 1
  23. while True:
  24. output_file = os.path.join(output_dir, f"{page}.json")
  25. url = f"https://dic.yajuvideo.in/comments?article_id={wid}&page={page}"
  27. if os.path.exists(output_file):
  28. with open(output_file, "r", encoding="utf-8") as f:
  29. data = json.load(f)
  31. if len(data["comments"]) >= 50:
  32. print(f"exists: {output_file}")
  33. page += 1
  34. continue
  35. else:
  36. print(f"update: {output_file}")
  37. else:
  38. print(url)
  40. r = requests.get(url, proxies=proxies, timeout=30)
  41. r.raise_for_status()
  43. soup = BeautifulSoup(r.text, "html.parser")
  44. thread = (
  45. soup.body
  46. .find("main")
  47. .select_one(".container .row")
  48. .find_all(recursive=False)[1]
  49. .select_one(".card")
  50. )
  52. title = thread.select_one(".card-header").get_text(strip=True)
  53. comments = thread.select(".card-body .comments-list > div[id^='comment-']")
  55. if len(comments) < 1:
  56. with open(os.path.join(output_dir, f"info.json"), "w", encoding="utf-8") as f:
  57. info = {
  58. "page_num_max": page - 1
  59. }
  60. json.dump(info, f, indent=2, ensure_ascii=False)
  61. break
  63. data = {
  64. "title": title,
  65. "comments": []
  66. }
  68. for i, c in enumerate(comments):
  69. comment = {}
  70. header = c.select_one(".comment-header")
  71. spans = header.find_all("span")
  73. name = spans[1].get_text(strip=True)
  74. date = spans[2].get_text(strip=True)
  76. body = c.select_one(".article-comment-body")
  78. text = normalize_comment(body)
  79. text = html.escape(text)
  80. text = text.replace("\n", "<br>")
  82. comment = {
  83. "name": name,
  84. "date": date,
  85. "text": text,
  86. }
  87. data["comments"].append(comment)
  89. with open(output_file, "w", encoding="utf-8") as f:
  90. json.dump(data, f, indent=2, ensure_ascii=False)
  91. time.sleep(0.1)
  93. page += 1
  95. def merge_json(wid):
  96. input_dir = os.path.join(ws_dir, f"{wid}")
  97. data_merged = {}
  99. with open(os.path.join(input_dir, "info.json"), "r", encoding="utf-8") as f:
  100. info = json.load(f)
  102. for i in range(info["page_num_max"]):
  103. with open(os.path.join(input_dir, f"{i+1}.json"), "r", encoding="utf-8") as f:
  104. data = json.load(f)
  105. if i == 0:
  106. data_merged = data
  107. else:
  108. data_merged["comments"] += data["comments"]
  110. with open(os.path.join(input_dir, "merged.json"), "w", encoding="utf-8") as f:
  111. json.dump(data_merged, f, indent=2, ensure_ascii=False)
  113. return
  115. def json_to_dat(wid):
  116. input_dir = os.path.join(ws_dir, f"{wid}")
  117. output_file = os.path.join(input_dir, f"all.dat")
  119. with open(os.path.join(input_dir, "merged.json"), "r", encoding="utf-8") as f:
  120. data_merged = json.load(f)
  122. lines = []
  123. for i, comment in enumerate(data_merged["comments"]):
  124. line = f"{comment["name"]}<> <>{comment["date"]}<> {comment["text"]}<> {data_merged["title"] if i == 0 else ""}"
  125. lines.append(line)
  127. with open(output_file, "w", encoding="utf-8") as f:
  128. f.write("\n".join(lines) + "\n")
  130. def json_to_html(wid):
  131. input_dir = os.path.join(ws_dir, f"{wid}")
  132. output_file = os.path.join(input_dir, f"all.html")
  134. with open(os.path.join(input_dir, "merged.json"), "r", encoding="utf-8") as f:
  135. data_merged = json.load(f)
  137. li_items = []
  139. for i, comment in enumerate(data_merged["comments"]):
  140. li_items.append(f""" <div id="comment-1" class="article-comment-item border-bottom p-3">
  141. <div class="comment-header small text-muted">
  142. <span class="fw-bold">{i+1}:</span>
  143. <span class="ms-1">{comment["name"]}</span>
  144. <span class="ms-2">{comment["date"]}</span>
  145. </div>
  146. <div class="article-comment-body mt-2">
  147. <p>{comment["text"]}</p>
  148. </div>
  149. </div>""")
  150. li_html = "\n".join(li_items)
  152. html = f"""<div class="container py-4">
  153. <div class="row">
  154. <div class="col-12 mb-4">
  155. <div class="card shadow-sm">
  156. <div class="card-header bg-white">
  157. <h2 class="card-title h4 mb-0">
  158. <i class="bi bi-book me-1"></i>
  159. {data_merged["title"]}
  160. </h2>
  161. </div>
  162. <div class="card-body p-0">
  163. <div class="comments-list">
  164. {li_html}
  165. </div>
  166. </div>
  167. </div>
  168. </div>
  169. </div>
  170. </div>"""
  172. with open(output_file, "w", encoding="utf-8") as f:
  173. f.write(html)
  175. def main():
  176. wid = 5
  178. save_json(wid)
  179. merge_json(wid)
  180. json_to_dat(wid)
  181. json_to_html(wid)
  183. main()
タグ: #野獣大百科

関連ノート

ゴMNT関連のノートまとめ
今までに作成したノートのリンク集です
@korosu_gmnt / 2026/02/10
ゴMNTパズピー集（X）
順次追加していきます 情報提供も募集しています
@korosu_gmnt / 2026/01/17
ゴMNTの転居先手掛かりまとめ
朝日ケーより 野獣大百科に載せるための簡単なまとめ
@korosu_gmnt / 2026/01/14
朝日ケー以前のゴMNTスレなど
@korosu_gmnt / 2026/01/13
Youtube関連スクリプトまとめ
@korosu_gmnt / 2026/01/10
ニコニコ関連スクリプトまとめ
@korosu_gmnt / 2026/01/05
野獣大百科の編集方針及びテンプレート集
私が記事を編集する際の方針とよく使うテンプレートを交えて備忘録として残します 内容は少しずつ増やしていきます
@korosu_gmnt / 2026/01/04
sm45784612の要約
ゴMNTが投稿した動画sm45784612（【※消されるかも】業務報告）のまとめです 元動画（無断転載版）：https://www.dailymotion.com/video/x9wrf32 これ公開したとして粗方有名な淫夢投...
@korosu_gmnt / 2025/12/31
風戸雄琉殺す
風戸雄琉殺す
@korosu_gmnt / 2025/12/31
最近追加されたノート