有没有合适的PYTHON抓字典的案例可参考？

stiggg

懂点PYTHON,想学学前人怎么爬字典的。目前只看到BT4BAIDU大大的开源，大大的代码很典范，可是代码四五年了，似乎网站也改版，具体细节的作用比较难揣摩了。不知道还有其它的比较好可参考学习？

毛小驴

顶起

你去哪里

我也想知道这个。

你去哪里

我也想知道这个。

scirem

1. #!/usr/bin/env python3
   * z6 @4 S5 ?  H9 y6 n
2. # -*- coding: utf-8 -*-
   
3. 
   
4. import os
   ; c' p5 [' I7 l0 n* z7 X6 P
5. import requests
   * N  v, |+ R; J1 I
6. import re
   
7. import time
   8 X, j0 T- I7 g: `
8. import sqlite3
   
9. 
   , R' @- u$ d- C+ [
10. from string import ascii_lowercase
    
11. 
    " ?2 {! {# q% P. f! {# j" E' u
12. from pathlib import Path
    + x& V5 b% E/ O" U2 G4 o
13. from urllib.parse import urljoin
    
14. from html.parser import HTMLParser
    ; k% X3 ]( a3 V9 E7 ?
15. from bs4 import BeautifulSoup
    * P* f4 ?, @. n  t& d+ g
16. 
    : A: y( P! o4 R+ z0 ]
17. webster_url = "https://www.merriam-webster.com/browse/dictionary/"
    + {0 I6 z6 P: }& F- z
18. oxford_url = "https://www.oxfordlearnersdictionaries.com/browse/english/"
    
19. macmillan_url = "https://www.macmillandictionary.com/browse/british/"
    ' u  J* k* z9 W0 i' `$ K
20. cambridge_url ="https://dictionary.cambridge.org/browse/english/"
    
21. 
    * P  k( b5 N" D; E" Y
22. cambridge_english_chinese_url = "https://dictionary.cambridge.org/browse/english-chinese-simplified/"
    " \) J8 E8 n) z5 W; b+ S; V
23. 
    
24. base_url = "https://dictionary.cambridge.org/search/direct/"
    
25. header = {"User-Agent": "Chrome/87.0.4280.88 Safari/537.36"}
    & A$ _+ E' z: A0 i; t0 ?. N( g
26. payload = {'datasetsearch': 'english-chinese-simplified', 'q': ""}
    
27. 
    ; s" e4 O: G5 [+ q
28. conn = sqlite3.connect("Cambridge Advanced Learner's English-Chinese Dictionary.db")
    7 {/ s% H5 J! N
29. 
    : M- U) n7 c9 X* _
30. conn.execute("""
    ( ~! O$ p% C9 b6 R+ O2 g% R. T
31. CREATE TABLE IF NOT EXISTS cambridge (
    
32.     entry TEXT PRIMARY KEY
    ! k- v' W0 c" F2 q- |
33.                UNIQUE
    
34.                NOT NULL,
    
35.     url   TEXT NOT NULL,
    
36.     html  TEXT NOT NULL,
    $ @, t* S0 f+ z/ R
37.     latex TEXT
    
38. );
    
39. """)
    
40. 
    
41. conn.commit()
    
42. 
    3 X  ]5 |7 S+ k9 t; i
43. 
    
44. def latexify(string):
    
45.     result = ""
    5 e2 x: x$ [, x; u% P* {
46.     trimmed_string = re.sub(r"\s{2,}", " ", string)
    
47.     for char in re.sub(r"\s+([^\w'"({[])", r"\1", trimmed_string):
    & z/ y( v1 F% |& z2 {
48.         if char == "%":
    # M9 V' d8 j* a$ J( K6 j. W% b, d
49.             result += r"\%"
    4 O/ a$ ?% p' H8 W: W* A
50.         elif char == "\":
    $ C9 s# y' l7 l7 U' V1 }+ }
51.             result += r"\textbackslash{}"
    6 A+ N/ n4 o9 F* _# [
52.         elif char == "$":
    
53.             result += r"\$"
    
54.         elif char == "#":
    ' a1 ?4 m0 v2 L3 A( |
55.             result += r"\#"
    
56.         elif char == "&":
    9 f1 T2 i1 A" `
57.             result += r"\&"
    7 e! z* n5 \+ h, c0 ], o1 G
58.         elif char == "{":
    , s, w( E/ y6 z: S
59.             result += r"\{"
    2 V6 a; S5 [: ]* J4 c
60.         elif char == "}":
    
61.             result += r"\}"
    
62.         elif char == "^":
    
63.             result += r"\^"
    2 s  B" X! x+ g& X
64.         elif char == "_":
    ! j. _- h7 V3 ~# O  O* X
65.             result += r"\_"
    
66.         elif char == "~":
    
67.             result += "r\textasciitilde{}"
    1 h, a0 H; ^1 F5 E- D* F1 C5 ^
68.         else:
    ! P- b9 p) E) r8 h7 \
69.             result += char
    / q) t% z% ]$ d: {; u# C8 u7 G6 l
70.     return result
    
71. 
    # m( z& p0 [5 m. a
72. def latexify_html(beautifulsoup_object):
    : R. m1 j# o1 g
73.     try:
    ) D; ], U" Z" z# E( p2 j( v( k0 \; O
74.         return latexify(re.sub(r"\s{2,}", " ", beautifulsoup_object.get_text().replace("\n", " ")).strip())
    
75.     except:
    2 M. H* \  m1 i3 r, z2 z
76.         return latexify(beautifulsoup_object)
    $ ^. W" p7 b! B. D+ B4 W
77. 
    " v- O) K& r$ O/ W9 r$ _/ a
78. 
    
79. class CambridgeDictionaryScraper:
    
80.     """ Scraper for Cambridge Dictionary """
    
81.     url_set = set() ## Shared by all instance
    " Q6 \2 {$ d0 O; v
82. 
    
83.     def __init__(self):
    
84.         for item in conn.execute("SELECT url FROM cambridge;"):
    " f0 e6 t0 |) w- T% K3 }8 c
85.             self.url_set.add(item[0])
    + t" ~! b. S2 R
86.         print("Already Downloaded " + str(len(self.url_set)) + " Words!")
    5 G4 F: K3 V0 ^8 g7 p: q
87. 
    
88. 
    
89.     def __del__(self):
    
90.         conn.commit()
    & E9 x/ ~* D# E& Z# P% [
91. 
    
92. 
    
93.     def get_word_page(self, url):
    ( }! f- x7 [3 o  a* w
94.         r = requests.get(url, headers=header)        
    
95.         bs_obj = BeautifulSoup(r.text, "html.parser")
    
96.         entry_tag = bs_obj.find("div", {"class": "entry"})
    
97.         if not entry_tag:
    
98.             entry_tag = bs_obj.find("div", {"class": "di-body"})
    
99.         if not entry_tag:
    
100.             ## Beta Words
     
101.             entry_tag = bs_obj.find("div", {"id": "entryContent"})
     
102.         if not entry_tag:
     0 P$ n: D8 X7 w
103.             entry_tag = bs_obj
     . }8 g0 R9 t7 K. S  C
104. 
     3 u0 l/ Z+ u+ }9 F1 ^
105.         if not entry_tag:
     
106.             entry_tag = bs_obj
     / j5 d. f" V+ d' T8 q8 C
107.         
     6 {$ c9 \5 G. |1 b; ?
108.         for tag in entry_tag.find_all("script"):
     
109.             tag.extract()
     6 s1 u5 V! Q- J
110.             
     
111.         result_string = str(entry_tag)
     
112.         return result_string
     # t% d- C# B  A9 O
113. 
       J. H3 x# D2 z- d, x9 s' O
114.    
     & ^$ o! w$ H1 Z6 H" a: {) ~
115.     def start(self, url):
     
116.         r = requests.get(url, headers=header)
     
117. 
     ( n+ |2 g# B! L. M; g; ]
118.         bs_obj = BeautifulSoup(r.text, "html.parser")
     1 ?, X2 c3 j- p# ~8 U7 x! f6 s
119. 
     3 Z7 X0 M# ^6 v. C% e) O+ N: b
120.         for li_tag in bs_obj.select("li.lpr-10"):
     
121.             child_url = urljoin(cambridge_url, li_tag.a.attrs["href"])
     
122.             print(child_url)
     
123.             self.find_child_entry(child_url)
     - n& ~' Y% g$ K  o  H( @- ^) b
124. 
     
125. 
     ; y% w2 J# _) k! z" B, {9 a
126.     def find_child_entry(self, url):
       g1 h* S, p6 N4 D9 _
127.         r = requests.get(url, headers=header)
     9 a0 [8 U3 y* {& h: S; B
128.         bs_obj = BeautifulSoup(r.text, "html.parser")
     
129.         for li_tag in bs_obj.select("li.t-i"):
     + z! y. p4 ]& d: h% m+ ?
130.             child_url = urljoin(url, li_tag.a.attrs["href"]).strip()
     
131.             child_text = li_tag.get_text().strip()
     9 L- H3 S' H4 h# a- d
132.             if "..." in child_text:
     
133.                 self.find_child_entry(child_url)
     , w* j: v0 m! c1 H$ H0 z+ b$ J
134.             else:
     7 \1 h: x: h: ]- b! e
135.                 if child_url in self.url_set:
     . }% n: u9 U; I8 O4 N) V# b( h, _
136.                     continue
     2 x, ^, z8 K+ N/ B; m# ?
137.                 print(child_text + "\t" + child_url)
     4 J5 X1 B+ y/ M, M
138.                 conn.execute("INSERT INTO cambridge (entry, url, html) VALUES (?, ?, ?);",
     1 _- n) ^. m3 f4 z; G1 ?* ?2 z
139.                 (child_text, child_url, self.get_word_page(child_url)))
     ) g/ m) I: J7 `! Y' z- l
140.                 conn.commit()
     
141.                 self.url_set.add(child_url)
     
142. 
     
143. 
     " Z  X, s7 h7 t% \- l
144. class CambridgeDictionaryExtractor():
     7 p% h  c0 v* ?1 S6 I" y- v& w& _
145.     def __init__(self, entry, entry_html = ""):
     - x7 t' R, R) p" O# I8 _  i
146.         self.entry = latexify(entry)
     + o8 V+ w6 W6 w
147.         self.entry_html = entry_html
     
148.         self.result = ""
     4 U. ^% _) j6 a; k: \4 f; n
149. 
     : y; f2 F- s' }" W; g3 o
150.     # def __del__(self):
     ; {5 b! f) l# ]5 c
151.     #     pass
     $ T4 M9 y; m! y: G7 T6 ]
152. 
     
153. 
       C; ]* [+ F+ \
154.     def extract(self):
     
155.         """
     & x& n3 b: k& O+ M2 j4 U4 a
156.         <div class="pr idiom-block">
     ; V. X) r" b& m- c9 i, w6 K
157.             <div class="idiom-block"></div>
     : M1 \- A  L  x0 }# v
158.         </div>
     # H* p, l( t- d* X* k6 h7 @. I
159.         """
     
160.         bs_obj = BeautifulSoup(self.entry_html, "html.parser")
     6 K# m. j$ Y" R6 r: D/ p
161.         self.result += "\\begin{entry}{" + self.entry + "}"
     
162.         for part_of_speech_tag in bs_obj.find_all("div", {"class": "entry-body__el"}):
     
163.             self.result += "\n\\begin{Partofspeech}\n" + self.process_part_of_speech(part_of_speech_tag) + "\n\\end{Partofspeech}"
     
164.         idiom_block = bs_obj.find("div", {"class": "idiom-block"})
     + |( U- P& u7 X
165.         if idiom_block:
     
166.             for idiom_block in idiom_block.find_all("div", {"class": "idiom-block"}):
     & M# w: R; c+ ?. `) c
167.                 self.result += "\n\\begin{idiom}" + self.process_idiom_block(idiom_block) + "\n\\end{idiom}"
     
168.         self.result += "\n\\end{entry}\n\n"
     
169. 
     
170.    
     
171.     def process_idiom_block(self, idiom_block):
     & Z$ R) x( H3 ~+ R
172.         result = ""
     ; N" M3 k" N/ P) B! t
173.         idiom_body = idiom_block.find("span", {"class": "idiom-body"})
     ' t) }$ V  Q( Y0 u. g
174.         if idiom_body:
     2 e# Z* z; n' O( N
175.             for sense_tag in idiom_body.find_all("div", {"class": "dsense"}):
     
176.                 result += self.process_sense_tag(sense_tag)
     * S& }7 y" Z2 B, T
177.         return result
     ( Y! s+ q- b+ ]% g
178. 
     
179.    
     
180. 
     
181.     def get_smart_vocabulary(self, smart_vocabulary_tag):
     
182.         result = ""
     6 `. r: a, v/ C* Z5 x
183.         for li_tag in smart_vocabulary_tag.find_all("li"):
       E* Q; p9 ?3 a4 U9 V
184.             result += "\\smart{" + latexify_html(li_tag) + "}\n"
     
185.         return "\n\\begin{smartvocabulary}\n" + result + "\\end{smartvocabulary}\n"
     2 R9 [* K9 x: t1 p$ m
186. 
     
187. 
     
188.     def process_part_of_speech(self, part_of_speech_tag):
     
189.         """
     
190.         <div class="entry-body__el">
     
191.             <div class="pos-header"></div>
     
192.             <div class="pos-body"></div>
     
193.             <div class="pr relativDiv"></div>
     ; U4 ~% L1 I: q% q' Y
194.         <div>
     
195.         """
     - u6 j( D5 [2 M# _
196.         result = ""
     
197.         header_tag = part_of_speech_tag.find("div", {"class": "pos-header"})
     
198.         if header_tag:
     3 z, ~( V9 r3 s
199.             result += self.process_part_of_speech_header(header_tag)
     * T+ h5 Q' S4 K) E
200.         body_tag = part_of_speech_tag.find("div", {"class": "pos-body"})
     
201.         if body_tag:
     
202.             result += self.process_part_of_speech_body(body_tag)
     ) e! V4 u1 F* D; G  X; t$ }, M) o
203.         pv_block_tag = part_of_speech_tag.find("div", {"class": "pv-block"})
     
204.         if pv_block_tag:
     2 G$ J! Q0 |/ Z' }2 J% q  U" C
205.             result += self.process_pv_block(pv_block_tag)
     
206.         return result.strip()
     * F; }* K- M, [
207. 
     
208.    
     
209.     def process_pv_block(self, tag):
     
210.         """
     
211.         <div class="pv-block">
     4 R/ E9 L5 ]6 R$ f7 z
212.             <div class="di-title"></div>
     
213.             <span clss="di-info"></span>
     : @. K7 Z- r2 Y# s& v# m+ n
214.             <span class="pv-body dpv-body">
     
215.                 <div class="pr dsense dsense-noh">
     # ?8 l) U. b$ V' F$ f5 }/ ~$ i
216.             <span>
     ! a5 t9 V- N1 y7 l2 x
217.         <div>
     * }3 w& ?& C& H' c
218.         """
     " D6 ]2 d7 J; j1 b
219.         result = ""
     # l. Q, y5 f5 E2 ~
220.         for item in tag.find_all("div",{"class", "sense-body"}):
     3 R  l0 X- q& {5 l6 f% \
221.             result += self.process_sense_body(item)
     
222.         return result
     
223. 
     
224. 
     ' R; d3 D2 q. F/ c
225.    
     3 U% {+ \6 n/ n* o6 Y: d
226.     def process_part_of_speech_header(self, header_tag):
     , |) Y+ R! P7 z  k" i4 T4 _
227.         result = ""
     / M  u3 h% n- k- W; X2 j
228.         # title_tag = header_tag.find("div", {"class": "di-title"})
     # T, G% S. m+ r! v( w/ f
229.         # if title_tag:
     
230.         #     result += process_header_title(title_tag)
     
231.         posgram_tag = header_tag.find("div", {"class": "posgram"})
     
232.         if posgram_tag:
     
233.             result += self.process_part_of_speech_grammar(posgram_tag)
     
234.         for pronunciation_tag in header_tag.find_all("span", {"class": "dpron-i"}):
     3 c" T/ `7 C: w$ }; ?3 Y
235.             result += self.process_pronunciation(pronunciation_tag)
     
236. 
     : A9 K) j: ~! V- Y
237.         return result.strip()
     ! z# j. \2 r0 V; y
238. 
     & h/ s3 A1 G+ t( T& w$ M. V
239. 
     9 x( ]+ a3 C+ ^  {0 o
240.     def process_header_title(self, title_tag):
     . A5 _- ^' m; j+ N
241.         ## <span class="hw dhw">record</span>
     2 X2 p9 f1 I# M! T% {( y$ J* q
242.         result = ""
     
243.         headword_tag = title_tag.find("span", {"class": "hw"})
     
244.         if headword_tag:
     
245.             result += "\\entry{" + latexify_html(headword_tag) + "}\n"
     + f) o- ?6 @; f4 s- L
246.         else:
     
247.             result += "\\entry{" + latexify_html(title_tag) + "}\n"
     
248.         return result
     
249. 
     
250.     def process_part_of_speech_grammar(self, posgram_tag):
     
251.         result = ""
     
252.         part_of_speech_tag = posgram_tag.find("span", {"class": "pos"})
     
253.         if part_of_speech_tag:
     
254.             result += "\n\\pos{" + latexify_html(part_of_speech_tag) + "}"
     - M5 X- z8 V7 t. R% h8 a4 ~! J
255.         gram_tag = posgram_tag.find("span", {"class": "gc"})
     
256.         if gram_tag:
     : [6 O- o0 t7 R
257.             result += "\n\\posgram{" + latexify_html(gram_tag)  + "}"
     8 C& E0 H; b( W9 B, g% y9 ]5 ^
258.         return result
     5 m1 U0 Y) n: C2 Z! Z- d( y4 z- E
259. 
     ( {) c, p# ?) p
260.     def process_pronunciation(self, pronunciation_tag):
     
261.         is_us_pronunciation = False
     4 v4 [/ c! X  |0 ~; s: l# {
262.         if "us" in pronunciation_tag.attrs["class"]:
     
263.             is_us_pronunciation = True
     
264.         result = ""
     . r* S2 K: {9 i4 o; F
265.         audio_tag = pronunciation_tag.find("source", {"type": "audio/mpeg"})
     ) R$ B. {9 {" E' H9 M# {/ z
266.         
     
267.         ipa_tag = pronunciation_tag.find("span", {"class": "ipa"})
     
268.         if ipa_tag:
     
269.             if is_us_pronunciation:
     2 v( V# q; @) q6 `+ [4 I, j/ S
270.                 result += "\n\\ipaus{" + latexify_html(ipa_tag) + "}"
     3 U7 d2 _; Z; o3 s" i- p
271.             else:
     
272.                 result += "\n\\ipauk{" + latexify_html(ipa_tag) + "}"
     
273.         if audio_tag:
     1 z& o$ Z  B$ O7 r
274.             audio_url = urljoin("https://dictionary.cambridge.org/", audio_tag.attrs["src"])
     
275.             result += "\n\pronuniation{" + audio_url + "}"
     8 }( V" o# J; `# ^
276.         return result
     0 [/ l- W+ V( D8 E1 F, T; E) }% O
277. 
     
278. 
     
279. 
     - [, Y* g& L3 `* d. J# R, g
280.     def process_sense_head(self, tag):
     
281.         text = latexify_html(tag)
     
282.         if "(" in text:
     
283.             left_bracket_index = text.index("(")
     % l7 A; |( |, W3 F% o* f
284.         else:
     + b3 u" B1 M5 F& m3 h
285.             left_bracket_index = 0
     $ g% b# J, h- L7 z- X5 h
286.         if ")" in text:
     ! S; ~1 e9 F9 W
287.             right_bracket_index = text.index(")")
     
288.         else:
     
289.             right_bracket_index = len(text)
     4 i8 W9 Q& M. `+ t9 u7 i" U% f8 y
290.         return "\n\\shortmeaning{" + text[left_bracket_index + 1:  right_bracket_index].strip() + "}"
     # n/ ~: J# j8 P( z! m
291. 
     / _' {! s7 W, C8 e( R
292. 
     1 {  [: w- O5 d& c: z
293.     def get_definition(self, tag):
     9 ]8 q% \- M  Y/ ]* a/ B
294.         result = ""
     
295.         for def_info_tag in tag.select("span.def-into"):
     7 ~( _+ Z0 \9 l' A4 R" T/ v
296.             result += latexify_html(def_info_tag)
     ( ]* U4 Q5 f8 P0 o" j6 J
297.         for def_tag in tag.select("div.def"):
     4 z0 d* a  P- k* v0 h5 `
298.             result += latexify_html(def_tag)
     
299.         return result
     
300. 
     
301. 
     : t7 J& T( C6 n4 l+ n
302.     def process_def_block(self, tag):
     
303.         result = ""
       `2 e8 e9 o  ?4 ]! ~* u: v& j* |; k
304.         def_tag = tag.find("div", {"class": "ddef_h"})
     
305.         if def_tag:
     
306.             result += "\n\\meaningen{" + self.get_definition(def_tag) + "}"
     % _0 N# {8 F$ T* X+ {+ W
307.         try:
     
308.             def_trans_tag = tag.select("span.trans")[0]
     
309.             if def_trans_tag:
     9 p/ j. L# L8 c+ C2 W
310.                 result += "\n\\meaningcn{" + latexify_html(def_trans_tag) + "}"
     / z, G0 {5 |  h$ _: Y$ E, U# m, X
311.         except:
       Y% m" B" v) ?
312.             pass
     - F- _2 A8 `  R" p
313.         for example_tag in tag.select("span.eg"):
     8 c' I& \+ e5 J7 [
314.             result += "\n\\example{" + latexify_html(example_tag) + "}"
     ) q; g$ f  ~- q
315.         return result
     6 ~( H- _) t6 n7 H: t7 p! Y
316. 
     
317. 
     6 W# b3 L% E4 c. q
318.     def process_phrase_block(self, phrase_block_tag):
     ' V! I3 e# H' n3 [- t
319.         """
     : o3 J& {7 Y: \1 W! S( W
320.         <div class="phrase-head dphrase_h">...</div>
     
321.         <div class="phrase-body dphrase_b">...</div>
     5 l, M* e0 L/ q0 D. S0 q/ t$ {2 X, I
322.         <div class="bb hax">...</div>
     7 w) O; ~: R' |) D( ^/ c( ^9 A
323.         """
     
324.         result = "\\begin{phrase}{"
     3 P7 t" k& e" {2 m
325.         result += phrase_block_tag.select("span.phrase-title")[0].get_text().strip() + "}"
     2 D1 `" T4 k% q- `# z
326.         return result + "\\end{pharse}\n"
     
327. 
     & z6 c9 k# a' [8 B. a
328.     def process_sense_body(self, tag):
     8 _/ Y) ^2 M) J: [# G
329.         """
     
330.         <div class="pr phrase-block dphrase-block">...</div>
     
331.         <div class="def-block ddef_block">...</div>
     + S. Y0 {: `8 K# U
332.         <div class="bb hax">...</div>
     
333.         """
     8 R# u2 Y; Y, u7 ]# R
334.         result = ""
     
335.         for def_block in tag.select("div.def-block"):
     ' t1 E8 A: J3 E$ ]* B: V- c
336.             result += self.process_def_block(def_block)
     
337.         for phrase_block in tag.select("div.pharse-block"):
     
338.             result += self.process_phrase_block(phrase_block)
     
339.         return result
     
340.         
     ' `  s2 ^6 e. z- W  H% o6 R
341.     def process_sense_tag(self, sense_tag):
     
342.         """
     ) ], X: }3 }( j  x5 _/ B
343.         <h3 class="dsense_h">...</h3>
     
344.         <div class="sense-body dsense_b">...</div>
     
345.         <div class="smartt daccord">...</div>           # Smart Vocabulary
     
346.         <div class="bb hax">...</div>
     7 ^4 q2 V% J0 z! I; l- k1 E3 L
347.         """
     3 h8 a8 ~! w" Q  I9 E
348.         result = ""
     - r. F3 C3 ]2 ^6 Y$ S
349.         sense_head_tag = sense_tag.find("h3", {"class": "dsense_h"})
     
350.         if sense_head_tag:
     ( @2 V" A' I, y9 _" @
351.             result += self.process_sense_head(sense_head_tag)
     ; ?$ ?+ X4 s- P5 O: a; b) n' Z
352.         for sense_body_tag in sense_tag.select("div.sense-body"):
     
353.             result += self.process_sense_body(sense_body_tag)
     
354.         for smart_vocabulary_tag in sense_tag.find_all("div", {"class": "smartt"}):
     " }$ ]5 T" C9 u& E9 [+ d
355.             result += self.get_smart_vocabulary(smart_vocabulary_tag)
     2 w- X: h) U+ U3 H8 N7 T
356.             
     
357.         return result
     
358. 
     " g& W2 s: `# O2 c, e+ y
359.     def process_part_of_speech_body(self, body_tag):
     
360.         """
     6 k5 |" a6 w5 L  i
361.         <div class="pr dsense">...</div>
     
362.         <div class="pr dsense">...</div>
     1 m: k$ s( L0 W- C, ~+ e
363.         """
     
364.         result = ""
     , e3 t1 A# @4 |4 c
365.         for sense_tag in body_tag.select("div.dsense"):
     , v' t. Y8 {5 L2 ^: I0 U* h- G
366.             result += self.process_sense_tag(sense_tag)
     3 Y! I9 D$ ]$ L1 H+ x$ C3 F, T, h
367.         return result
     
368.    
     3 A" h% D2 A0 j4 V! r% v; b
369. 
     
370. if __name__ == "__main__":
     / W5 _/ N2 J. E: U3 r
371.     string = ""
       W6 K- @& f8 O( q$ ^7 g
372.     CambridgeDictionaryScraper().start(cambridge_english_chinese_url)
     
373.     for row in conn.execute("SELECT entry, url, html FROM cambridge;"):
     ; @' ~2 X9 A& t5 b- D9 u: `
374.         entry = row[0]
     
375.         print(entry)
     
376.         url = row[1]
     # D; K( w/ M* X9 r4 n+ a0 E
377.         html = row[2]
     
378.         record = CambridgeDictionaryExtractor(entry, entry_html=html)
     
379.         record.extract()
     
380.         string += record.result + "\n"
     - M* J$ u7 v$ N, t+ i% v
381.         #print(record.result)
     
382. 
     
383.     with open("./final.tex", "w", encoding="utf-8") as f:
     ) J$ k* S* e; m; d* }9 ?1 i; j$ M
384.         try:
     ) t( T* Q4 _5 c- u/ C
385.             for char in ascii_lowercase:
     
386.                 string = string.replace("\\begin{entry}{" + char, "\\section{" + char + "}\n\n\\begin{entry}{" + char, 1)
     
387.         except:
     
388.             pass
     
389.         
     
390.         f.write(string.replace("", ""))
     
391. 
     
392. 
     
393.         

复制代码