有没有合适的PYTHON抓字典的案例可参考？

stiggg

懂点PYTHON,想学学前人怎么爬字典的。目前只看到BT4BAIDU大大的开源，大大的代码很典范，可是代码四五年了，似乎网站也改版，具体细节的作用比较难揣摩了。不知道还有其它的比较好可参考学习？

毛小驴

顶起

你去哪里

我也想知道这个。

你去哪里

我也想知道这个。

scirem

1. #!/usr/bin/env python3
   * `- s& f( ]( ]* W3 c! J' A
2. # -*- coding: utf-8 -*-
   
3. 
   7 K- }, q; X) O2 N& [2 V. J
4. import os
   2 W) L: ], y6 @/ E
5. import requests
   3 f- {$ t1 Z, j+ j
6. import re
   
7. import time
   
8. import sqlite3
   9 W% s% Q9 S6 s* `6 M; R) r6 h4 F
9. 
   $ c2 R! k/ T3 @$ f
10. from string import ascii_lowercase
    $ b) Y1 V( L# h; i$ u0 B5 J4 h
11. 
    
12. from pathlib import Path
    6 ?' @; X5 S7 d* x- ?
13. from urllib.parse import urljoin
    
14. from html.parser import HTMLParser
    ! N5 \: U$ V+ @+ O$ f9 a' {
15. from bs4 import BeautifulSoup
    
16. 
    : ?$ q* i3 ~! |7 d1 h6 Z, X
17. webster_url = "https://www.merriam-webster.com/browse/dictionary/"
    
18. oxford_url = "https://www.oxfordlearnersdictionaries.com/browse/english/"
    2 g' L0 t6 W/ N
19. macmillan_url = "https://www.macmillandictionary.com/browse/british/"
    
20. cambridge_url ="https://dictionary.cambridge.org/browse/english/"
    , q# U; B2 U, H( z
21. 
    / E( n( U' ^# ]* C$ O
22. cambridge_english_chinese_url = "https://dictionary.cambridge.org/browse/english-chinese-simplified/"
    6 o  Z9 M$ `2 a" N8 G
23. 
    ' w; H& A, B! g- Q3 O  b) K
24. base_url = "https://dictionary.cambridge.org/search/direct/"
    
25. header = {"User-Agent": "Chrome/87.0.4280.88 Safari/537.36"}
    1 `7 D( z8 K4 s- B3 ~) q
26. payload = {'datasetsearch': 'english-chinese-simplified', 'q': ""}
    
27. 
    
28. conn = sqlite3.connect("Cambridge Advanced Learner's English-Chinese Dictionary.db")
    
29. 
    / l4 n, }2 j- B4 \1 k! W2 k
30. conn.execute("""
      t4 j" |7 B: \2 ]
31. CREATE TABLE IF NOT EXISTS cambridge (
    
32.     entry TEXT PRIMARY KEY
    
33.                UNIQUE
    
34.                NOT NULL,
    
35.     url   TEXT NOT NULL,
    8 ?8 N& U; K# h' f, n+ w) ^
36.     html  TEXT NOT NULL,
    # ~: p+ p9 T* \5 W. Q$ Y* G. g
37.     latex TEXT
    
38. );
    $ U  \1 \- c# ^  o3 ]
39. """)
    $ l: K! F% \4 q
40. 
    + X) H- _7 k7 U. b5 p
41. conn.commit()
    
42. 
    
43. 
    4 w7 J+ J, S  E  i4 O2 M
44. def latexify(string):
    
45.     result = ""
    # D  L/ Z, i' u
46.     trimmed_string = re.sub(r"\s{2,}", " ", string)
    
47.     for char in re.sub(r"\s+([^\w'"({[])", r"\1", trimmed_string):
    ) n% k9 [1 g9 k
48.         if char == "%":
    
49.             result += r"\%"
    6 c4 k0 A; H: U# a
50.         elif char == "\":
    3 {  k3 v' N  J" L- B7 H6 P
51.             result += r"\textbackslash{}"
    
52.         elif char == "$":
    
53.             result += r"\$"
    7 z8 G. [9 \' n
54.         elif char == "#":
    2 X5 _' q7 y% v$ s
55.             result += r"\#"
    9 ]5 }8 D, O% o( ~
56.         elif char == "&":
    9 Q$ {( w/ G% O
57.             result += r"\&"
    
58.         elif char == "{":
    ) b) N% C- K6 e$ H9 r
59.             result += r"\{"
    & d- t7 ]8 |; F% n. L% s! Q, h8 `
60.         elif char == "}":
    
61.             result += r"\}"
    
62.         elif char == "^":
    
63.             result += r"\^"
    ) z5 B$ j" t! Y3 ?* H. L) ]
64.         elif char == "_":
    8 B% m5 l6 ~3 H' o- n, u0 s
65.             result += r"\_"
    
66.         elif char == "~":
    
67.             result += "r\textasciitilde{}"
    
68.         else:
    
69.             result += char
    . ], L( i2 a5 ]* R* Y6 u
70.     return result
    
71. 
    , Q: ]3 i9 X  V
72. def latexify_html(beautifulsoup_object):
    7 Y( E9 |5 {) M4 F, I/ t
73.     try:
    2 N, `/ }; N5 i8 W& y. l6 e
74.         return latexify(re.sub(r"\s{2,}", " ", beautifulsoup_object.get_text().replace("\n", " ")).strip())
    % l$ v  J/ L9 [5 F$ m5 k. q2 C
75.     except:
    - \( `2 c$ ]4 y$ C9 u7 ]
76.         return latexify(beautifulsoup_object)
    . W) h  i! X' ~7 D+ H- l# H! b
77. 
    2 z4 x/ f* Z1 V  {
78. 
    5 Z8 H, g' Z; |- R+ k
79. class CambridgeDictionaryScraper:
    
80.     """ Scraper for Cambridge Dictionary """
    + e: T0 @: v1 f
81.     url_set = set() ## Shared by all instance
    9 U4 C( \: f5 I  j, Y6 S- \3 Z  ~
82. 
    " S  U$ j+ T; Q- B# e8 Z7 S1 T
83.     def __init__(self):
    % F" q* R  p3 f: U& o/ h/ Y  t5 U& M7 `
84.         for item in conn.execute("SELECT url FROM cambridge;"):
    
85.             self.url_set.add(item[0])
    
86.         print("Already Downloaded " + str(len(self.url_set)) + " Words!")
    
87. 
    + f" O$ p$ O" ?0 U- O! _
88. 
    ( B! }5 r* p8 g) g
89.     def __del__(self):
    / ?) Q5 j/ r, L) S4 l
90.         conn.commit()
    
91. 
    
92. 
    - x5 j3 T% f) I# L
93.     def get_word_page(self, url):
    
94.         r = requests.get(url, headers=header)        
    2 Y! \" T0 C' d  P9 D0 k
95.         bs_obj = BeautifulSoup(r.text, "html.parser")
    
96.         entry_tag = bs_obj.find("div", {"class": "entry"})
    , [5 B. _- m6 w8 l7 B
97.         if not entry_tag:
    # b) o" n+ h, {; X: s
98.             entry_tag = bs_obj.find("div", {"class": "di-body"})
    ' E, U$ ^- T7 ^
99.         if not entry_tag:
    
100.             ## Beta Words
     9 S1 l3 i$ Z' P" ~9 y, S
101.             entry_tag = bs_obj.find("div", {"id": "entryContent"})
     
102.         if not entry_tag:
       ~3 F$ j+ G/ `" V3 m
103.             entry_tag = bs_obj
     1 R. _9 @5 o+ K- F! m5 y
104. 
     8 O" \0 D6 }8 H0 K: ]7 x
105.         if not entry_tag:
     6 K  V. M/ n$ b$ T6 a# t
106.             entry_tag = bs_obj
     
107.         
     
108.         for tag in entry_tag.find_all("script"):
     
109.             tag.extract()
     
110.             
     ) j# v6 @9 `# q1 z& Z; q, R) d1 n( j
111.         result_string = str(entry_tag)
     - H! A6 E- Z% B! ~2 c' V# M. e
112.         return result_string
     
113. 
     & f- ^9 s$ l+ n0 P5 x5 y
114.    
     ' i$ h$ o5 h) a7 A3 R. k" Q/ I1 G
115.     def start(self, url):
     
116.         r = requests.get(url, headers=header)
     . c/ n1 Q* ]( P
117. 
     ! l4 ~$ I+ m: J2 H) ~
118.         bs_obj = BeautifulSoup(r.text, "html.parser")
     
119. 
     
120.         for li_tag in bs_obj.select("li.lpr-10"):
     ; z$ E9 G( q" Q, l2 Q: d
121.             child_url = urljoin(cambridge_url, li_tag.a.attrs["href"])
     7 K3 F0 ?! f% {6 X# w- N
122.             print(child_url)
     2 t$ J: k  d5 Z- a" V  t, {
123.             self.find_child_entry(child_url)
     + V2 n3 u0 r5 u! s
124. 
     2 T/ [2 u, G" J. x8 n4 X. L
125. 
     
126.     def find_child_entry(self, url):
     8 [/ q+ {8 N4 B- s+ Y! O
127.         r = requests.get(url, headers=header)
     ! {' b$ I5 L1 C! |; Y
128.         bs_obj = BeautifulSoup(r.text, "html.parser")
     : B' W" l. T6 z  m$ g& O& g
129.         for li_tag in bs_obj.select("li.t-i"):
       L. {& w: j. s: g1 ~
130.             child_url = urljoin(url, li_tag.a.attrs["href"]).strip()
     
131.             child_text = li_tag.get_text().strip()
     3 g/ C8 v4 q: q& E5 ]& K
132.             if "..." in child_text:
     5 F9 P1 k" @4 Z% R
133.                 self.find_child_entry(child_url)
     & H" z+ J3 s2 Q, }8 W; ~: v
134.             else:
     2 g" o) E6 i- w3 G- y
135.                 if child_url in self.url_set:
     
136.                     continue
     : `, F5 y0 h; G, y
137.                 print(child_text + "\t" + child_url)
     7 g% R" B% C' i
138.                 conn.execute("INSERT INTO cambridge (entry, url, html) VALUES (?, ?, ?);",
     ' r! M( w  s3 b: P/ ~- R
139.                 (child_text, child_url, self.get_word_page(child_url)))
     
140.                 conn.commit()
     
141.                 self.url_set.add(child_url)
     6 b0 y2 Q' G+ q% v! q/ ~& N
142. 
     
143. 
     
144. class CambridgeDictionaryExtractor():
     7 w' j/ X) d4 v/ w( o# S  l
145.     def __init__(self, entry, entry_html = ""):
     
146.         self.entry = latexify(entry)
     . I7 ~5 c6 k/ S' H1 Q# f/ L
147.         self.entry_html = entry_html
     
148.         self.result = ""
     
149. 
     ; R* b# `- i- t: T: W& S
150.     # def __del__(self):
     ( e  X. i- Z3 m6 H
151.     #     pass
     ( Z7 B/ h8 d* K: e; Z0 s
152. 
     # K0 L$ [4 A5 H1 ]9 A% N
153. 
     * _& S2 ~4 I% g2 ~5 {
154.     def extract(self):
     
155.         """
     + `) Y) `( e6 W4 @4 Z0 Z9 y
156.         <div class="pr idiom-block">
     
157.             <div class="idiom-block"></div>
     
158.         </div>
     
159.         """
     
160.         bs_obj = BeautifulSoup(self.entry_html, "html.parser")
     4 Y; `9 o( @+ b& d) E3 U) V' ~
161.         self.result += "\\begin{entry}{" + self.entry + "}"
     ; }# Q: Y( ~! V: |
162.         for part_of_speech_tag in bs_obj.find_all("div", {"class": "entry-body__el"}):
     
163.             self.result += "\n\\begin{Partofspeech}\n" + self.process_part_of_speech(part_of_speech_tag) + "\n\\end{Partofspeech}"
     8 Z5 |" A3 }7 ~
164.         idiom_block = bs_obj.find("div", {"class": "idiom-block"})
     0 h7 ]6 I# X% I8 X1 @' {2 ]" V
165.         if idiom_block:
     ; f# v5 o7 A  p4 G' t5 Y( M' P
166.             for idiom_block in idiom_block.find_all("div", {"class": "idiom-block"}):
     ) H7 ^+ `) \8 I( d
167.                 self.result += "\n\\begin{idiom}" + self.process_idiom_block(idiom_block) + "\n\\end{idiom}"
     
168.         self.result += "\n\\end{entry}\n\n"
     
169. 
     
170.    
     
171.     def process_idiom_block(self, idiom_block):
     0 c! ~9 ?1 w$ |: `" {
172.         result = ""
     # c* Z1 Q5 g7 p' \/ H& C+ z
173.         idiom_body = idiom_block.find("span", {"class": "idiom-body"})
     
174.         if idiom_body:
     8 n! E/ v0 Y# P5 m6 `! \
175.             for sense_tag in idiom_body.find_all("div", {"class": "dsense"}):
     
176.                 result += self.process_sense_tag(sense_tag)
     " K. j9 q3 b& K. \; d
177.         return result
     
178. 
     
179.    
     ; ?5 K/ ~! T4 F
180. 
     
181.     def get_smart_vocabulary(self, smart_vocabulary_tag):
     " O' Q9 U( Q$ v, a0 j
182.         result = ""
     ; S# P( I4 Z9 r8 v: Q
183.         for li_tag in smart_vocabulary_tag.find_all("li"):
     
184.             result += "\\smart{" + latexify_html(li_tag) + "}\n"
     
185.         return "\n\\begin{smartvocabulary}\n" + result + "\\end{smartvocabulary}\n"
     
186. 
     
187. 
     # n" S- z5 k/ u! J3 ?2 d, w0 @
188.     def process_part_of_speech(self, part_of_speech_tag):
     
189.         """
     + {/ I$ t. \0 W1 V& b8 _3 k
190.         <div class="entry-body__el">
     
191.             <div class="pos-header"></div>
     : s- h; k( @9 |7 Q: E
192.             <div class="pos-body"></div>
     / c' w  I1 G( \( q! H. Q0 e
193.             <div class="pr relativDiv"></div>
     2 C' R0 i; x. w+ I: s2 T3 n' y
194.         <div>
     " K! T- E) ~! Z  E  ?+ S% K! y
195.         """
     
196.         result = ""
     
197.         header_tag = part_of_speech_tag.find("div", {"class": "pos-header"})
     ) |$ K. b6 n; L& P1 i
198.         if header_tag:
     
199.             result += self.process_part_of_speech_header(header_tag)
     1 o  F( G& [$ ]$ i
200.         body_tag = part_of_speech_tag.find("div", {"class": "pos-body"})
     ' g/ u5 K% n9 D* Z! ]3 Q& q
201.         if body_tag:
     5 H' h# K( i- ]0 K& \4 `. Q4 O
202.             result += self.process_part_of_speech_body(body_tag)
     
203.         pv_block_tag = part_of_speech_tag.find("div", {"class": "pv-block"})
     
204.         if pv_block_tag:
     ' E  M6 F" }$ N/ H  A
205.             result += self.process_pv_block(pv_block_tag)
     
206.         return result.strip()
     * u  \" u, N8 z0 c. O: e- ~& J
207. 
     ( g# J" T* d8 {" l( p/ k
208.    
     ) Z# I3 Y( D: t* z: U
209.     def process_pv_block(self, tag):
     
210.         """
     
211.         <div class="pv-block">
     
212.             <div class="di-title"></div>
     , Y5 C, w7 `2 r; w( G
213.             <span clss="di-info"></span>
     : c) H5 z: x2 x* e) p3 V$ e  V) ^
214.             <span class="pv-body dpv-body">
     & h: ]8 A8 S1 G& T- I
215.                 <div class="pr dsense dsense-noh">
     / Q, N$ U. Z, ^  L
216.             <span>
     
217.         <div>
     5 f( G6 w! D: `- W- C
218.         """
     
219.         result = ""
     5 r8 m' k4 [+ e
220.         for item in tag.find_all("div",{"class", "sense-body"}):
     
221.             result += self.process_sense_body(item)
     
222.         return result
     ' e+ g8 _5 f8 m" }
223. 
     
224. 
     
225.    
     0 e" D7 L2 U) o3 A$ u
226.     def process_part_of_speech_header(self, header_tag):
     4 E1 I* x3 }$ c9 r3 G, k0 T
227.         result = ""
     
228.         # title_tag = header_tag.find("div", {"class": "di-title"})
     ; Q# L' O( @: G4 H
229.         # if title_tag:
       k) r' ]! D6 `- F2 v/ J$ g2 T
230.         #     result += process_header_title(title_tag)
     / E; j5 n  {' G" O- o1 _+ C
231.         posgram_tag = header_tag.find("div", {"class": "posgram"})
     ' W5 t9 U. H9 a5 k
232.         if posgram_tag:
     6 u/ g" t. @. v) K
233.             result += self.process_part_of_speech_grammar(posgram_tag)
     , f, ^/ Q" O, n% u2 G5 h0 N3 R7 G
234.         for pronunciation_tag in header_tag.find_all("span", {"class": "dpron-i"}):
     , ~! B1 h1 a& A
235.             result += self.process_pronunciation(pronunciation_tag)
     # o9 U& {& ^) z- H  a8 d
236. 
     
237.         return result.strip()
     + B( E8 r; S" T! {) w, \5 R
238. 
     
239. 
     - o; Y4 {2 |% K* V% x: E
240.     def process_header_title(self, title_tag):
     * ]" M& R2 c, T) h# ~# ?
241.         ## <span class="hw dhw">record</span>
     
242.         result = ""
     
243.         headword_tag = title_tag.find("span", {"class": "hw"})
     
244.         if headword_tag:
     $ x% X8 i# b% `9 D5 j
245.             result += "\\entry{" + latexify_html(headword_tag) + "}\n"
     + o* b' }! @' m3 E
246.         else:
     4 F' s) N1 {, v
247.             result += "\\entry{" + latexify_html(title_tag) + "}\n"
     ! q) X0 V& f# j& l) v8 P1 m
248.         return result
     
249. 
     ) n) d3 h9 m2 I
250.     def process_part_of_speech_grammar(self, posgram_tag):
     1 n2 v. J2 W$ ]! o
251.         result = ""
     
252.         part_of_speech_tag = posgram_tag.find("span", {"class": "pos"})
     ( e3 R. X  w& C" q* w4 e
253.         if part_of_speech_tag:
     
254.             result += "\n\\pos{" + latexify_html(part_of_speech_tag) + "}"
     
255.         gram_tag = posgram_tag.find("span", {"class": "gc"})
     
256.         if gram_tag:
     ; g! X6 l' y* }2 L) P+ e" k- ^0 u
257.             result += "\n\\posgram{" + latexify_html(gram_tag)  + "}"
     
258.         return result
     . [) L# X3 Y2 J! C
259. 
     
260.     def process_pronunciation(self, pronunciation_tag):
     
261.         is_us_pronunciation = False
     
262.         if "us" in pronunciation_tag.attrs["class"]:
       I7 y4 ~- N& {, H0 ^
263.             is_us_pronunciation = True
     ! m  y. ?6 i' c8 F, ]( b6 K. P
264.         result = ""
     4 U& \& X5 o6 }0 m
265.         audio_tag = pronunciation_tag.find("source", {"type": "audio/mpeg"})
     & d) s2 h  _# b& N. J# }
266.         
     6 l2 X* M  o) W, m
267.         ipa_tag = pronunciation_tag.find("span", {"class": "ipa"})
     ' p; k$ v: j1 O4 v7 x  c
268.         if ipa_tag:
     $ ~6 o0 p+ A" A  F% p, p! z) _7 s
269.             if is_us_pronunciation:
     
270.                 result += "\n\\ipaus{" + latexify_html(ipa_tag) + "}"
       ^3 m/ {6 D9 |: Q
271.             else:
     4 y- e6 m9 q1 ~) I
272.                 result += "\n\\ipauk{" + latexify_html(ipa_tag) + "}"
     
273.         if audio_tag:
     
274.             audio_url = urljoin("https://dictionary.cambridge.org/", audio_tag.attrs["src"])
     
275.             result += "\n\pronuniation{" + audio_url + "}"
     
276.         return result
     " @- j+ k) O1 m+ _
277. 
     " F2 i8 u" Y& r; Z, v
278. 
     3 M- Z* {7 k4 u( b' ]6 i, e4 z
279. 
     
280.     def process_sense_head(self, tag):
     
281.         text = latexify_html(tag)
     
282.         if "(" in text:
       H  n4 x7 i9 s6 }% Q" A2 V% Y
283.             left_bracket_index = text.index("(")
     
284.         else:
     * j, C6 C7 P& l/ b; t6 e
285.             left_bracket_index = 0
     
286.         if ")" in text:
     6 l, l. ^2 E6 b7 O( h
287.             right_bracket_index = text.index(")")
     
288.         else:
     & s6 J' k; \1 L! o" c# I
289.             right_bracket_index = len(text)
     
290.         return "\n\\shortmeaning{" + text[left_bracket_index + 1:  right_bracket_index].strip() + "}"
     
291. 
     
292. 
     
293.     def get_definition(self, tag):
     
294.         result = ""
     2 t# r# q) P9 v9 j$ d) l
295.         for def_info_tag in tag.select("span.def-into"):
     ( {) k  e5 Q4 B; A/ Y3 u
296.             result += latexify_html(def_info_tag)
     
297.         for def_tag in tag.select("div.def"):
     5 D/ D5 b7 j, A. ]  d
298.             result += latexify_html(def_tag)
     " E5 J+ H/ }* y- v0 m/ `
299.         return result
     0 k- z' X8 M+ k) L, [
300. 
     
301. 
     / h5 s6 h. ~4 K
302.     def process_def_block(self, tag):
     
303.         result = ""
     ' d+ q/ d) {8 w! ^. I$ c- b
304.         def_tag = tag.find("div", {"class": "ddef_h"})
     
305.         if def_tag:
     ) K# a7 c% Q; Z% S6 N
306.             result += "\n\\meaningen{" + self.get_definition(def_tag) + "}"
     
307.         try:
     
308.             def_trans_tag = tag.select("span.trans")[0]
     
309.             if def_trans_tag:
     
310.                 result += "\n\\meaningcn{" + latexify_html(def_trans_tag) + "}"
     
311.         except:
     
312.             pass
     ; i: H1 a9 G: {1 m
313.         for example_tag in tag.select("span.eg"):
     + n, g1 O3 I' Y! M$ P; q& e
314.             result += "\n\\example{" + latexify_html(example_tag) + "}"
     
315.         return result
     
316. 
     
317. 
     / X, ]1 o" K& [. _
318.     def process_phrase_block(self, phrase_block_tag):
     
319.         """
     0 z* t/ \  d/ N1 _2 z/ O
320.         <div class="phrase-head dphrase_h">...</div>
     9 E" t  f, P5 _& L
321.         <div class="phrase-body dphrase_b">...</div>
     + m0 k* y/ F2 e! j7 p) z5 m
322.         <div class="bb hax">...</div>
     
323.         """
     ) H: R- v. O' P. C$ k) a6 g
324.         result = "\\begin{phrase}{"
     . ^, j6 }4 D: {* w* K8 Q
325.         result += phrase_block_tag.select("span.phrase-title")[0].get_text().strip() + "}"
     
326.         return result + "\\end{pharse}\n"
     
327. 
     , a* Q# E5 ?& p4 C$ O; D# `+ G8 A
328.     def process_sense_body(self, tag):
     ) B7 o/ A( M+ w/ J; m
329.         """
     
330.         <div class="pr phrase-block dphrase-block">...</div>
     
331.         <div class="def-block ddef_block">...</div>
     ! ]  r+ t) d5 \( C) D6 p8 L
332.         <div class="bb hax">...</div>
     : ]+ ?# p& M2 E* o  \( K
333.         """
     
334.         result = ""
     
335.         for def_block in tag.select("div.def-block"):
     
336.             result += self.process_def_block(def_block)
     
337.         for phrase_block in tag.select("div.pharse-block"):
     
338.             result += self.process_phrase_block(phrase_block)
     + ]: T7 G6 i0 q
339.         return result
     ( t/ r2 L: u: t: N7 a* W
340.         
     - Z* Y0 ^8 ?9 V7 U
341.     def process_sense_tag(self, sense_tag):
     ' n) B: F6 N. l8 Y8 a1 z
342.         """
     : l. V1 _# G9 j4 g
343.         <h3 class="dsense_h">...</h3>
     ) Y/ [) j+ {  R0 [& D
344.         <div class="sense-body dsense_b">...</div>
     * T% e4 z9 c- ^8 h/ s- L
345.         <div class="smartt daccord">...</div>           # Smart Vocabulary
     
346.         <div class="bb hax">...</div>
     * Q9 ^( k* ], ~( y; d
347.         """
     8 b$ y3 X* n) k3 q: O" s3 k! @
348.         result = ""
     
349.         sense_head_tag = sense_tag.find("h3", {"class": "dsense_h"})
     ( O+ Q4 f' p# w$ Q6 `. n# G
350.         if sense_head_tag:
     
351.             result += self.process_sense_head(sense_head_tag)
     8 h- q2 X- O7 \4 ^1 }
352.         for sense_body_tag in sense_tag.select("div.sense-body"):
     
353.             result += self.process_sense_body(sense_body_tag)
     
354.         for smart_vocabulary_tag in sense_tag.find_all("div", {"class": "smartt"}):
     
355.             result += self.get_smart_vocabulary(smart_vocabulary_tag)
     
356.             
     
357.         return result
     6 Y  K, E/ S+ b2 F) R
358. 
     & y7 o; Q. a- C" B2 L% K) Z" X
359.     def process_part_of_speech_body(self, body_tag):
     
360.         """
     # r, N/ ~- m% W  p! I
361.         <div class="pr dsense">...</div>
     8 }  R- I, e( f, L% [0 n5 l1 d, L
362.         <div class="pr dsense">...</div>
     : |, }9 a( e9 ~( ^/ L
363.         """
     
364.         result = ""
     
365.         for sense_tag in body_tag.select("div.dsense"):
     , i( N: Y# m, F' M
366.             result += self.process_sense_tag(sense_tag)
     2 m2 f4 T8 ?. X- w
367.         return result
     
368.    
     ! w8 d0 `* P- [2 l4 X' Z+ H6 v
369. 
     ) U7 C4 h9 y% c' I) k# S
370. if __name__ == "__main__":
     / O1 ]8 R# o2 C; `) S* K( ?
371.     string = ""
     ; j* g" I) v* O0 ~
372.     CambridgeDictionaryScraper().start(cambridge_english_chinese_url)
     ' y9 u2 z/ w. |6 W. e, D! x3 @8 j
373.     for row in conn.execute("SELECT entry, url, html FROM cambridge;"):
     - t' S- b6 ^6 i9 n8 h
374.         entry = row[0]
     3 m7 Z( b' U1 o. y! D  t
375.         print(entry)
     3 v3 I/ I8 M& J; D! P
376.         url = row[1]
     $ U4 p  I/ ?+ W, g
377.         html = row[2]
     
378.         record = CambridgeDictionaryExtractor(entry, entry_html=html)
     + ^6 L) m; X7 {( E- }# F
379.         record.extract()
     4 D9 R- a& G% ?% M; N
380.         string += record.result + "\n"
     % c; a, y3 S, ^8 x7 J: V8 c5 W
381.         #print(record.result)
     
382. 
     : Z) Z/ f1 I% e3 x! |
383.     with open("./final.tex", "w", encoding="utf-8") as f:
     ' q: W" ~6 c4 n2 g, {
384.         try:
     
385.             for char in ascii_lowercase:
     9 Q% J) A4 ?' [+ ]: T$ ~; m" O
386.                 string = string.replace("\\begin{entry}{" + char, "\\section{" + char + "}\n\n\\begin{entry}{" + char, 1)
     ' P- p. b- o9 u2 s: g8 R
387.         except:
     
388.             pass
     % L% }) a& g% u3 a5 C' ^: `
389.         
     
390.         f.write(string.replace("", ""))
     
391. 
     6 R" \# H2 ]. b# s; R: _
392. 
     . |( `1 f8 @9 u/ g
393.         

复制代码