PubChem数据库爬虫笔记

1
<tr>
2
   <th class="inf">Formula</th>
3
   <td colspan="4">C15H14O3</td>
4
</tr>
5
<tr>
6
   <th class="inf">Mw</th>
7
   <td colspan="4">242.09429431</td>
8
</tr>

1
# Looks in a table for a row with the given label and gets the corresponding value from the same row
2
# Find any table row <tr> where the first cell (whether it's a <th> or <td>) exactly matches the label.
3
def get_text_label_in_table(driver, label):
4
    try:
5
        row = driver.find_element(
6
            By.XPATH,
7
            f"//table//tr[normalize-space(.//th[1] | .//td[1])='{label}']"
8
        )
9
        return row.find_element(By.XPATH, './td[1]').text.strip()
10
    except NoSuchElementException:
11
        return None
12

13
# Parser for Knapsack
14
# Extracts Formula and Mw from chemical entry pages on knapsackfamily.com using table-based scraping
15
def parse_knapsack(driver):
16
    formula = get_text_label_in_table(driver, 'Formula')
17
    weight = get_text_label_in_table(driver, 'Mw')
18
    if weight is None:
19
        weight = get_text_label_in_table(driver, 'Molecular weight')
20
    return weight, formula

1
<tr>
2
    <td width="70%" align="right">Molecular Weight: &nbsp;</td>
3
    <td width="30%" align="center">154.03</td>
4
</tr>

1
# Parser for NPASS
2
# Extracts Formula and Molecular Weight from npass.bidd.group using <dt>/<dd> tags and fallbacks to table parsing if needed
3
def parse_npass(driver):
4

5
    # Try extracting formula from <dt>/<dd>
6
    try:
7
        formula = driver.find_element(
8
            By.XPATH,
9
            "//dt[contains(normalize-space(),'Molecular Formula')]/following-sibling::dd[1]"
10
        ).text.strip()
11
    except NoSuchElementException:
12
        formula = None
13

14
    # Try extracting weight from <dt>/<dd>
15
    try:
16
        weight = driver.find_element(
17
            By.XPATH,
18
            "//dt[contains(normalize-space(),'Molecular Weight')]/following-sibling::dd[1]"
19
        ).text.strip()
20
    except NoSuchElementException:
21
        weight = None
22

23
    # NPASS often uses a <table class="table_with_border">…</table> for Mw;
24
    # if the dt/dd lookup failed or returned '0', fall back to grabbing from the table.
25
    if not weight or weight == '0':
26
        try:
27
            weight = driver.find_element(
28
                By.XPATH,
29
                "//table[contains(@class,'table_with_border')]"
30
                "//tr[td[1][contains(normalize-space(.),'Molecular Weight')]]/td[2]"
31
            ).text.strip()
32
        except NoSuchElementException:
33
            weight = None
34

35
    return weight, formula

1
{
2
  "entities": {
3
    "Qxxx": {
4
      "claims": {
5
        "P274": [...],  // Formula
6
        "P2067": [...]  // Molecular weight
7
      }
8
    }
9
  }
10
}

1
# Parser for Wikidata
2
# Uses the Wikidata API to extract:
3
#   P274: chemical formula
4
#   P2067: molecular weight
5
# Handles potential nested dictionary responses
6
def parse_wikidata(entity_id):
7
    import urllib.request, json
8

9
    api_url = (
10
        'https://www.wikidata.org/w/api.php'
11
        '?action=wbgetentities&ids=%s&props=claims&format=json' % entity_id
12
    )
13
    try:
14
        with urllib.request.urlopen(api_url, timeout=PAGE_LOAD_TIMEOUT) as f:
15
            data = json.load(f)
16
        claims = data['entities'][entity_id]['claims']
17
        formula = None
18
        weight = None
19
        if 'P274' in claims:
20
            formula = claims['P274'][0]['mainsnak']['datavalue']['value']
21
        if 'P2067' in claims:
22
            weight = claims['P2067'][0]['mainsnak']['datavalue']['value']
23
            # Wikidata returns a dict {'amount': '+<value>', 'unit': ...}; extract the numeric amount
24
            if isinstance(weight, dict):
25
                raw_amount = weight.get('amount')
26
                weight = raw_amount.lstrip('+') if raw_amount is not None else None
27
        return weight, formula
28
    except Exception as e:
29
        print(f"WARNING: failed to fetch Wikidata {entity_id}: {e}", file=sys.stderr)
30
        return None, None