有没有合适的PYTHON抓字典的案例可参考？

stiggg

懂点PYTHON,想学学前人怎么爬字典的。目前只看到BT4BAIDU大大的开源，大大的代码很典范，可是代码四五年了，似乎网站也改版，具体细节的作用比较难揣摩了。不知道还有其它的比较好可参考学习？

毛小驴

顶起

你去哪里

我也想知道这个。

你去哪里

我也想知道这个。

scirem

1. #!/usr/bin/env python3
   
2. # -*- coding: utf-8 -*-
   1 I( X9 ~0 o: H0 m1 q
3. 
   ) I: y5 J- ~/ e+ K, e( r. m& e
4. import os
   
5. import requests
   4 W5 D; l1 C; g
6. import re
   
7. import time
   " P7 [% I" T0 I0 w8 y. ]5 F1 h
8. import sqlite3
   5 A9 I# ~( M' V1 \
9. 
   - C* ~4 a/ {+ O
10. from string import ascii_lowercase
    ! R# U$ e* m% u) g
11. 
    4 H: Q; p' |1 @/ t* a# [9 Q
12. from pathlib import Path
    & T' t5 }; S  \2 X" S% r! y- I
13. from urllib.parse import urljoin
    
14. from html.parser import HTMLParser
    
15. from bs4 import BeautifulSoup
    ) C$ Z1 P7 ^' P# r/ ?
16. 
    
17. webster_url = "https://www.merriam-webster.com/browse/dictionary/"
    5 s9 j: f& Z8 U# e
18. oxford_url = "https://www.oxfordlearnersdictionaries.com/browse/english/"
    
19. macmillan_url = "https://www.macmillandictionary.com/browse/british/"
    0 d9 p4 C, K3 n# ?7 J
20. cambridge_url ="https://dictionary.cambridge.org/browse/english/"
    
21. 
    
22. cambridge_english_chinese_url = "https://dictionary.cambridge.org/browse/english-chinese-simplified/"
    ; [6 S* u6 h& T! ]( H8 A
23. 
    
24. base_url = "https://dictionary.cambridge.org/search/direct/"
    % \7 B4 M/ Q* E+ {5 f- y
25. header = {"User-Agent": "Chrome/87.0.4280.88 Safari/537.36"}
    ) L# T' `: G5 J  f  n7 e
26. payload = {'datasetsearch': 'english-chinese-simplified', 'q': ""}
    
27. 
    
28. conn = sqlite3.connect("Cambridge Advanced Learner's English-Chinese Dictionary.db")
    
29. 
    4 `2 W5 d7 L" ~
30. conn.execute("""
    ' E2 z- s* F, V! j  A+ V) Z4 p
31. CREATE TABLE IF NOT EXISTS cambridge (
    
32.     entry TEXT PRIMARY KEY
    
33.                UNIQUE
    3 a' C& c$ a4 c8 o2 q# ^$ L$ z
34.                NOT NULL,
    . u8 v+ F& f- s4 M. p
35.     url   TEXT NOT NULL,
    6 H/ U) ]' B) i/ g; b8 ~* }
36.     html  TEXT NOT NULL,
    4 u- u$ l0 o$ @+ ~; d, j! k1 ]
37.     latex TEXT
    
38. );
    
39. """)
    & n8 w. r: U" d9 O- l# O; Y1 [, k
40. 
    1 N/ m+ d' \& F7 F; F1 L* i! h$ |
41. conn.commit()
      z; i' k( q6 [
42. 
    1 K2 G8 `9 \: O  o8 c
43. 
    
44. def latexify(string):
    
45.     result = ""
    4 E2 y; E+ ^9 n6 f! j0 u, b
46.     trimmed_string = re.sub(r"\s{2,}", " ", string)
    
47.     for char in re.sub(r"\s+([^\w'"({[])", r"\1", trimmed_string):
    , I( G4 Y! `9 H& ~; g$ b
48.         if char == "%":
    1 ?2 v% \  U( B
49.             result += r"\%"
    
50.         elif char == "\":
    
51.             result += r"\textbackslash{}"
    
52.         elif char == "$":
    $ v0 J: v4 X' ]: M4 z2 w5 K
53.             result += r"\$"
    
54.         elif char == "#":
    ; m- `) t' G+ ?$ _. d8 R- O- ~
55.             result += r"\#"
    & ]! c; o& Z) ?8 N" l
56.         elif char == "&":
    $ q; p7 |8 I6 K2 k
57.             result += r"\&"
    
58.         elif char == "{":
    4 E9 }! {" Q  ?: D7 [, e) X) M
59.             result += r"\{"
    
60.         elif char == "}":
    
61.             result += r"\}"
    1 X+ E) M8 r. g6 N2 S4 G7 b# s
62.         elif char == "^":
    + p, f" q$ `7 U
63.             result += r"\^"
    6 O: Y" y1 t3 G3 z, S2 A
64.         elif char == "_":
    4 U: Y; r/ M4 F9 t' k
65.             result += r"\_"
    
66.         elif char == "~":
    . G, r- P5 k: V  U3 N; }
67.             result += "r\textasciitilde{}"
    . Y" ]* t* b6 a( S% ]
68.         else:
    
69.             result += char
    
70.     return result
    : ^  c7 G: z, H# }# t, @
71. 
    
72. def latexify_html(beautifulsoup_object):
    
73.     try:
    3 Z" z* h) \7 r# h
74.         return latexify(re.sub(r"\s{2,}", " ", beautifulsoup_object.get_text().replace("\n", " ")).strip())
    # b9 W, Z9 T1 e  n
75.     except:
    * ^5 \( ~# t- e$ n
76.         return latexify(beautifulsoup_object)
    * B/ B0 I& N6 @& ]: c) n2 n
77. 
    " w9 u" X' f& D3 j0 z2 b# L$ m  C
78. 
    
79. class CambridgeDictionaryScraper:
    
80.     """ Scraper for Cambridge Dictionary """
    # D5 A& A0 X0 F/ r6 X) D5 K+ m) z
81.     url_set = set() ## Shared by all instance
    
82. 
    6 M8 }" ?* o2 q' O4 g) L( R  ]
83.     def __init__(self):
    
84.         for item in conn.execute("SELECT url FROM cambridge;"):
    
85.             self.url_set.add(item[0])
    $ C2 C& c2 D7 p9 Z+ U
86.         print("Already Downloaded " + str(len(self.url_set)) + " Words!")
    . e( B+ |2 h' D# i& C
87. 
    7 k/ Y: f- y, X# \  s0 G
88. 
    
89.     def __del__(self):
    ; b- [# O0 P0 H& b! O
90.         conn.commit()
    $ q5 v/ J  |, _& y( Q$ F- A% X, o
91. 
    ( _& r" X0 d( f# q. U' A: x
92. 
    $ B. j, U$ D1 l2 s" r% _4 v0 l
93.     def get_word_page(self, url):
    
94.         r = requests.get(url, headers=header)        
    
95.         bs_obj = BeautifulSoup(r.text, "html.parser")
    
96.         entry_tag = bs_obj.find("div", {"class": "entry"})
    
97.         if not entry_tag:
    ! {) ^% [1 J. V& D) O* o* |
98.             entry_tag = bs_obj.find("div", {"class": "di-body"})
    # y8 i  i. V. `% P; m
99.         if not entry_tag:
    
100.             ## Beta Words
     - {+ L6 _1 m4 {* l; ^! A* n
101.             entry_tag = bs_obj.find("div", {"id": "entryContent"})
     
102.         if not entry_tag:
     ' c6 ^, J  W" @% i$ x
103.             entry_tag = bs_obj
     
104. 
     $ n3 o( J4 ^* c  [$ H' K
105.         if not entry_tag:
     2 f0 |4 |0 W$ `" S8 k' \3 P
106.             entry_tag = bs_obj
     
107.         
     8 _% h6 P3 f( i
108.         for tag in entry_tag.find_all("script"):
     6 T! Q1 |. ^; _5 W9 Z( u" B* J6 o
109.             tag.extract()
     * _8 \7 R9 @5 u5 J) b6 Q: n
110.             
     
111.         result_string = str(entry_tag)
     
112.         return result_string
     6 ^0 l2 ^0 G6 y" E1 F6 k/ B
113. 
       c+ p3 Q  B1 j- F+ v
114.    
       `5 x6 S; ]3 t* }* a4 e
115.     def start(self, url):
     
116.         r = requests.get(url, headers=header)
     1 i1 O7 |2 c* k0 U
117. 
     
118.         bs_obj = BeautifulSoup(r.text, "html.parser")
     ; W+ h  ]2 D" ^1 v! @& }3 ^6 T
119. 
     7 i" \! S' [% V, d8 u4 Q
120.         for li_tag in bs_obj.select("li.lpr-10"):
     
121.             child_url = urljoin(cambridge_url, li_tag.a.attrs["href"])
     
122.             print(child_url)
     
123.             self.find_child_entry(child_url)
     
124. 
     ( O& O1 B8 H2 @4 c6 g8 d
125. 
     1 l! W7 k' K; ?2 m% \: T  `
126.     def find_child_entry(self, url):
     
127.         r = requests.get(url, headers=header)
     8 Z1 ?. H+ r9 j* Z* A& [- O" s
128.         bs_obj = BeautifulSoup(r.text, "html.parser")
     " W4 L- {+ ?+ R, T2 X) q
129.         for li_tag in bs_obj.select("li.t-i"):
     
130.             child_url = urljoin(url, li_tag.a.attrs["href"]).strip()
     ) T: `' N& A  j' i2 U1 L
131.             child_text = li_tag.get_text().strip()
       Q* X" J8 J6 F. S9 J9 [* k8 h
132.             if "..." in child_text:
     
133.                 self.find_child_entry(child_url)
     2 {4 h2 V+ K4 a3 S+ G; z: g
134.             else:
     
135.                 if child_url in self.url_set:
     
136.                     continue
     2 k5 P0 y( ~& j- J0 j+ y- K
137.                 print(child_text + "\t" + child_url)
     ; P1 `0 B' f* M0 `6 [  Z6 V" O4 a
138.                 conn.execute("INSERT INTO cambridge (entry, url, html) VALUES (?, ?, ?);",
     : _# R, p9 J/ M% b3 \/ q
139.                 (child_text, child_url, self.get_word_page(child_url)))
     
140.                 conn.commit()
     
141.                 self.url_set.add(child_url)
     
142. 
     7 T6 |& A' j- R/ a
143. 
     
144. class CambridgeDictionaryExtractor():
     : Z: s2 w6 [% B- m% v$ D# x
145.     def __init__(self, entry, entry_html = ""):
     , D. v2 T7 z) g. \- x# {
146.         self.entry = latexify(entry)
     
147.         self.entry_html = entry_html
     6 H* p9 U4 h& c
148.         self.result = ""
     ! r1 J) F  b4 h7 K; I
149. 
     " Z, S: o: [" t$ L5 f& C7 b
150.     # def __del__(self):
     
151.     #     pass
     
152. 
     
153. 
     8 f$ J5 k; K0 `' Z9 x
154.     def extract(self):
     % N1 n% L$ h9 F4 J
155.         """
     
156.         <div class="pr idiom-block">
     
157.             <div class="idiom-block"></div>
     
158.         </div>
     $ ^  h9 V0 y/ R7 D1 d
159.         """
     
160.         bs_obj = BeautifulSoup(self.entry_html, "html.parser")
     
161.         self.result += "\\begin{entry}{" + self.entry + "}"
     
162.         for part_of_speech_tag in bs_obj.find_all("div", {"class": "entry-body__el"}):
     
163.             self.result += "\n\\begin{Partofspeech}\n" + self.process_part_of_speech(part_of_speech_tag) + "\n\\end{Partofspeech}"
     3 Q+ }1 G; Z7 |7 X3 Y
164.         idiom_block = bs_obj.find("div", {"class": "idiom-block"})
     
165.         if idiom_block:
     
166.             for idiom_block in idiom_block.find_all("div", {"class": "idiom-block"}):
     ; J+ k$ u: R6 y* [2 R) O
167.                 self.result += "\n\\begin{idiom}" + self.process_idiom_block(idiom_block) + "\n\\end{idiom}"
     
168.         self.result += "\n\\end{entry}\n\n"
     
169. 
     
170.    
     
171.     def process_idiom_block(self, idiom_block):
     & W6 D' z8 Z* h8 m1 p) W
172.         result = ""
     
173.         idiom_body = idiom_block.find("span", {"class": "idiom-body"})
     2 y8 g/ j+ X" ]9 K( A
174.         if idiom_body:
     
175.             for sense_tag in idiom_body.find_all("div", {"class": "dsense"}):
       J4 K* w) V, ~" e6 R1 Q! T! E" _- q
176.                 result += self.process_sense_tag(sense_tag)
     
177.         return result
     
178. 
     
179.    
     ; e2 b( S; X  D- L: w
180. 
     , j# v0 D5 e6 R# \9 l, r1 Z
181.     def get_smart_vocabulary(self, smart_vocabulary_tag):
     # B  g1 E3 Q: ^* e! }+ g0 P  H
182.         result = ""
     
183.         for li_tag in smart_vocabulary_tag.find_all("li"):
     
184.             result += "\\smart{" + latexify_html(li_tag) + "}\n"
     + z' e" S3 S7 K
185.         return "\n\\begin{smartvocabulary}\n" + result + "\\end{smartvocabulary}\n"
     
186. 
     
187. 
     
188.     def process_part_of_speech(self, part_of_speech_tag):
     
189.         """
     7 \& c0 _  _5 v3 }
190.         <div class="entry-body__el">
     
191.             <div class="pos-header"></div>
     
192.             <div class="pos-body"></div>
     
193.             <div class="pr relativDiv"></div>
     * [/ C3 Q" b+ w( T# n7 T
194.         <div>
     
195.         """
     . A+ o1 B- F6 g( _
196.         result = ""
     
197.         header_tag = part_of_speech_tag.find("div", {"class": "pos-header"})
     $ A2 j' S5 ^; D: [' i0 j, R
198.         if header_tag:
     " d, l& W! c. N9 s& I
199.             result += self.process_part_of_speech_header(header_tag)
     # `. L. S0 _5 @$ ^# q' p- W' ]
200.         body_tag = part_of_speech_tag.find("div", {"class": "pos-body"})
     7 b. }( W, E( j/ L% i6 L
201.         if body_tag:
     8 U! `" J' C! f7 ]4 [5 N- e  G
202.             result += self.process_part_of_speech_body(body_tag)
     
203.         pv_block_tag = part_of_speech_tag.find("div", {"class": "pv-block"})
     * _2 p5 F: e8 Q) t
204.         if pv_block_tag:
     ; l  u( ]! _# z' e1 O, o4 L# M
205.             result += self.process_pv_block(pv_block_tag)
     ) k: a/ I  a1 m, U
206.         return result.strip()
     ; o+ S3 r5 X" s3 b+ A
207. 
     
208.    
     
209.     def process_pv_block(self, tag):
     4 O$ D: y; m* y2 j. v' `  d
210.         """
     ( h$ E+ K* T3 L! J
211.         <div class="pv-block">
     + e) v7 Z1 X; M1 T- T& W
212.             <div class="di-title"></div>
     
213.             <span clss="di-info"></span>
     
214.             <span class="pv-body dpv-body">
     
215.                 <div class="pr dsense dsense-noh">
     
216.             <span>
     * x+ }- G, t0 c: D$ M
217.         <div>
     
218.         """
     7 Q, U2 H) [: {, }  S; x9 I; I; i
219.         result = ""
     + m4 w: z: Q1 l
220.         for item in tag.find_all("div",{"class", "sense-body"}):
     $ P, ^7 D. y4 r+ g* o! [
221.             result += self.process_sense_body(item)
     1 z6 C! E- N% V: Y: i3 W
222.         return result
     ' z/ Q0 o/ _9 n3 _8 L+ b8 r
223. 
     / V, F' T/ V7 k& H/ I. u
224. 
     
225.    
     2 D* N" b) U* g$ s( C7 _7 s
226.     def process_part_of_speech_header(self, header_tag):
     
227.         result = ""
     ) A! [, l  ~9 @+ l3 r4 h
228.         # title_tag = header_tag.find("div", {"class": "di-title"})
     5 T+ j/ _/ F& ?/ ^' q; Q+ S
229.         # if title_tag:
     : h+ L2 E( u/ q' o$ t3 \: {8 y3 W9 f, Q
230.         #     result += process_header_title(title_tag)
     
231.         posgram_tag = header_tag.find("div", {"class": "posgram"})
     8 O7 M3 \% n7 Z- J
232.         if posgram_tag:
     4 u( S/ }9 U: m, o
233.             result += self.process_part_of_speech_grammar(posgram_tag)
     / G7 J$ r; p* O& g
234.         for pronunciation_tag in header_tag.find_all("span", {"class": "dpron-i"}):
     
235.             result += self.process_pronunciation(pronunciation_tag)
     ' ~5 v9 N, W$ Q. P& b. S
236. 
     ! \6 s1 S' e3 R9 T3 ~
237.         return result.strip()
       X8 M- P. A8 I! ]! ^9 X
238. 
     
239. 
     . n; w" P9 ~% w) V& Y. M8 e
240.     def process_header_title(self, title_tag):
     5 n4 H' C. f- J6 f
241.         ## <span class="hw dhw">record</span>
     0 Q! G8 K5 e3 D  y& L
242.         result = ""
     
243.         headword_tag = title_tag.find("span", {"class": "hw"})
     ! \8 Z" \$ r6 v& i+ c* q  J/ P
244.         if headword_tag:
     
245.             result += "\\entry{" + latexify_html(headword_tag) + "}\n"
     - @4 h; h1 i% Q5 e+ a0 @* E/ C! z% j
246.         else:
     
247.             result += "\\entry{" + latexify_html(title_tag) + "}\n"
     $ u" M4 w  f# S3 ^
248.         return result
     ) {9 T# H0 w5 s1 ~: x
249. 
     4 ]; }& O/ _* N5 n5 x
250.     def process_part_of_speech_grammar(self, posgram_tag):
     
251.         result = ""
     
252.         part_of_speech_tag = posgram_tag.find("span", {"class": "pos"})
     9 r7 V* h" a: ?% e
253.         if part_of_speech_tag:
     - u* V( h3 }& l& J* E; E
254.             result += "\n\\pos{" + latexify_html(part_of_speech_tag) + "}"
     / H8 h5 o; y+ j6 [* r
255.         gram_tag = posgram_tag.find("span", {"class": "gc"})
     ! @6 ]3 t" t( X( v0 x6 C
256.         if gram_tag:
     
257.             result += "\n\\posgram{" + latexify_html(gram_tag)  + "}"
     
258.         return result
     
259. 
     
260.     def process_pronunciation(self, pronunciation_tag):
     
261.         is_us_pronunciation = False
     ) ^# }# h4 i5 k1 f% h, `0 X
262.         if "us" in pronunciation_tag.attrs["class"]:
     8 ]+ u( O$ I# B
263.             is_us_pronunciation = True
     
264.         result = ""
     
265.         audio_tag = pronunciation_tag.find("source", {"type": "audio/mpeg"})
     
266.         
     5 ^  `5 s* C1 L% T" W' y) @
267.         ipa_tag = pronunciation_tag.find("span", {"class": "ipa"})
     , ~2 ~' o. F% N. v& x3 i% }
268.         if ipa_tag:
     
269.             if is_us_pronunciation:
     . D  V) W2 N6 Q* E- v: x
270.                 result += "\n\\ipaus{" + latexify_html(ipa_tag) + "}"
     1 B) C+ W* P/ p) f: W5 v
271.             else:
     
272.                 result += "\n\\ipauk{" + latexify_html(ipa_tag) + "}"
     * a( k) ?$ O6 [. T/ l8 V) o( S/ y
273.         if audio_tag:
     
274.             audio_url = urljoin("https://dictionary.cambridge.org/", audio_tag.attrs["src"])
     
275.             result += "\n\pronuniation{" + audio_url + "}"
     
276.         return result
     0 O5 K7 b- z2 T
277. 
     
278. 
       G! j$ Y; D  M, r& z6 @8 P* y  R3 F
279. 
     4 t/ y1 \( I- S0 k
280.     def process_sense_head(self, tag):
     
281.         text = latexify_html(tag)
     8 i9 c: Y- K/ L) A( _2 F  I% p
282.         if "(" in text:
     ' @# O5 h% ?+ h9 t' F6 y7 x
283.             left_bracket_index = text.index("(")
     4 d* H6 N' S  d/ M# X
284.         else:
     
285.             left_bracket_index = 0
     
286.         if ")" in text:
     & g; L6 A& S% |
287.             right_bracket_index = text.index(")")
     
288.         else:
     " \1 {6 G2 U! l7 Y# c! H; e. y
289.             right_bracket_index = len(text)
     
290.         return "\n\\shortmeaning{" + text[left_bracket_index + 1:  right_bracket_index].strip() + "}"
     " r! U" R' }2 J9 ]0 o
291. 
     4 s& P& Q+ Z( A; d1 ]3 @0 l0 j+ U
292. 
     ( O1 v! z8 I7 p0 q9 f
293.     def get_definition(self, tag):
     ) q- n6 k& r: C) z
294.         result = ""
     
295.         for def_info_tag in tag.select("span.def-into"):
     
296.             result += latexify_html(def_info_tag)
     
297.         for def_tag in tag.select("div.def"):
     . @( {  o! L  [
298.             result += latexify_html(def_tag)
     
299.         return result
     # |# k: L& z9 k$ f+ ~/ u
300. 
     / ]$ w+ b/ S& z9 y
301. 
     
302.     def process_def_block(self, tag):
     7 m. o% a2 `2 f4 a* k  p7 i
303.         result = ""
     - o; A. C  `' i$ K
304.         def_tag = tag.find("div", {"class": "ddef_h"})
     
305.         if def_tag:
     
306.             result += "\n\\meaningen{" + self.get_definition(def_tag) + "}"
     9 W6 L* @. o4 }! w  a, f* M
307.         try:
     
308.             def_trans_tag = tag.select("span.trans")[0]
     
309.             if def_trans_tag:
     
310.                 result += "\n\\meaningcn{" + latexify_html(def_trans_tag) + "}"
     
311.         except:
     
312.             pass
     8 A) i  P/ \! l% X, u5 G/ Z3 t# \
313.         for example_tag in tag.select("span.eg"):
     ( S# z3 l: m! g0 n" U! c# P% n
314.             result += "\n\\example{" + latexify_html(example_tag) + "}"
     3 j& r' ^. C' U+ L/ K: L4 O
315.         return result
     
316. 
     
317. 
     9 W" r. g+ j6 s. `
318.     def process_phrase_block(self, phrase_block_tag):
     9 ~6 B2 u2 x4 c
319.         """
     
320.         <div class="phrase-head dphrase_h">...</div>
     / F2 {  U  n7 ~3 z1 p
321.         <div class="phrase-body dphrase_b">...</div>
     
322.         <div class="bb hax">...</div>
     $ ]2 k* v0 K7 v8 V. X
323.         """
     3 b) v' _, |  ~1 W
324.         result = "\\begin{phrase}{"
     
325.         result += phrase_block_tag.select("span.phrase-title")[0].get_text().strip() + "}"
       B0 ?4 e3 N+ C$ ]
326.         return result + "\\end{pharse}\n"
     5 w- ^5 S. A& S1 Z# k. G
327. 
     ' l3 @# g. ]6 H# w9 G
328.     def process_sense_body(self, tag):
     
329.         """
     1 ^; C8 z* p  k0 z2 }
330.         <div class="pr phrase-block dphrase-block">...</div>
     
331.         <div class="def-block ddef_block">...</div>
     $ s- @6 f& |) f- R' D
332.         <div class="bb hax">...</div>
     
333.         """
     
334.         result = ""
     8 t+ y* X, g, F1 S  o1 o% i
335.         for def_block in tag.select("div.def-block"):
     # {8 P2 k- i/ x, j4 r
336.             result += self.process_def_block(def_block)
     
337.         for phrase_block in tag.select("div.pharse-block"):
     
338.             result += self.process_phrase_block(phrase_block)
     
339.         return result
     
340.         
     
341.     def process_sense_tag(self, sense_tag):
     . o' w6 p+ h+ f+ w- \( j
342.         """
     
343.         <h3 class="dsense_h">...</h3>
     4 P6 L1 Z/ {, D( |
344.         <div class="sense-body dsense_b">...</div>
     
345.         <div class="smartt daccord">...</div>           # Smart Vocabulary
     
346.         <div class="bb hax">...</div>
     
347.         """
     
348.         result = ""
     
349.         sense_head_tag = sense_tag.find("h3", {"class": "dsense_h"})
     5 K1 ^+ z% v, s# ]9 k, \/ J
350.         if sense_head_tag:
     " l3 P3 Z0 ^  ?- i: @) {/ v
351.             result += self.process_sense_head(sense_head_tag)
     
352.         for sense_body_tag in sense_tag.select("div.sense-body"):
     ! j" f$ |  |% x! D
353.             result += self.process_sense_body(sense_body_tag)
     
354.         for smart_vocabulary_tag in sense_tag.find_all("div", {"class": "smartt"}):
     
355.             result += self.get_smart_vocabulary(smart_vocabulary_tag)
     . }5 B" {  d$ h1 t# l4 i+ o
356.             
     ( M8 x4 `* Q+ W- \' b9 }
357.         return result
     
358. 
     
359.     def process_part_of_speech_body(self, body_tag):
     ! U7 W# k! f# L* Y( \
360.         """
     5 S. N) O2 r) r$ G  X
361.         <div class="pr dsense">...</div>
     " P2 h* q' u! k" s
362.         <div class="pr dsense">...</div>
     7 q7 {- ^7 P3 N" Q9 x9 z% e4 U) |
363.         """
     % z* ^9 h* A2 F& R
364.         result = ""
     3 X4 s! z6 r4 c5 f- k
365.         for sense_tag in body_tag.select("div.dsense"):
     
366.             result += self.process_sense_tag(sense_tag)
     6 b# l1 _9 s* K0 T
367.         return result
     % r6 f6 d, @' s
368.    
     7 w1 W/ L; u7 L* Z4 p
369. 
     ! a: o& S& _, T: N  W* p% s' D+ K
370. if __name__ == "__main__":
     ; z, i2 q: s0 o
371.     string = ""
     " Z1 I$ a0 r) a( m$ n2 H1 v! }
372.     CambridgeDictionaryScraper().start(cambridge_english_chinese_url)
     
373.     for row in conn.execute("SELECT entry, url, html FROM cambridge;"):
     8 y) T1 @# |1 N- h
374.         entry = row[0]
     
375.         print(entry)
     / T/ N% w: G; M6 S1 Y, o. A
376.         url = row[1]
     
377.         html = row[2]
     
378.         record = CambridgeDictionaryExtractor(entry, entry_html=html)
     ' e! V1 c# ~0 I' y
379.         record.extract()
     $ A) w& V' Q' ?, o
380.         string += record.result + "\n"
     0 }$ g9 Q5 Q/ x
381.         #print(record.result)
     
382. 
     
383.     with open("./final.tex", "w", encoding="utf-8") as f:
     
384.         try:
     
385.             for char in ascii_lowercase:
     3 F$ B$ S7 D3 O  ]+ w4 O2 j* `
386.                 string = string.replace("\\begin{entry}{" + char, "\\section{" + char + "}\n\n\\begin{entry}{" + char, 1)
       p4 K1 l. _5 }% H) U# Q
387.         except:
     ) I( w7 F* _+ ]+ u
388.             pass
     
389.         
     % m/ H( l8 {) n& J' h6 g
390.         f.write(string.replace("", ""))
     2 H) ~% M2 j% N6 t" t" O5 N5 d
391. 
     
392. 
     
393.         

复制代码