Initial commit: sfda_parser Frappe app
This commit is contained in:
commit
7113bf1867
21
.editorconfig
Normal file
21
.editorconfig
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# Root editor config file
|
||||||
|
root = true
|
||||||
|
|
||||||
|
# Common settings
|
||||||
|
[*]
|
||||||
|
end_of_line = lf
|
||||||
|
insert_final_newline = true
|
||||||
|
trim_trailing_whitespace = true
|
||||||
|
charset = utf-8
|
||||||
|
|
||||||
|
# python, js indentation settings
|
||||||
|
[{*.py,*.js,*.vue,*.css,*.scss,*.html}]
|
||||||
|
indent_style = tab
|
||||||
|
indent_size = 4
|
||||||
|
max_line_length = 99
|
||||||
|
|
||||||
|
# JSON files - mostly doctype schema files
|
||||||
|
[{*.json}]
|
||||||
|
insert_final_newline = false
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
124
.eslintrc
Normal file
124
.eslintrc
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
{
|
||||||
|
"env": {
|
||||||
|
"browser": true,
|
||||||
|
"node": true,
|
||||||
|
"es2022": true
|
||||||
|
},
|
||||||
|
"parserOptions": {
|
||||||
|
"sourceType": "module"
|
||||||
|
},
|
||||||
|
"extends": "eslint:recommended",
|
||||||
|
"rules": {
|
||||||
|
"indent": "off",
|
||||||
|
"brace-style": "off",
|
||||||
|
"no-mixed-spaces-and-tabs": "off",
|
||||||
|
"no-useless-escape": "off",
|
||||||
|
"space-unary-ops": ["error", { "words": true }],
|
||||||
|
"linebreak-style": "off",
|
||||||
|
"quotes": ["off"],
|
||||||
|
"semi": "off",
|
||||||
|
"camelcase": "off",
|
||||||
|
"no-unused-vars": "off",
|
||||||
|
"no-console": ["warn"],
|
||||||
|
"no-extra-boolean-cast": ["off"],
|
||||||
|
"no-control-regex": ["off"],
|
||||||
|
},
|
||||||
|
"root": true,
|
||||||
|
"globals": {
|
||||||
|
"frappe": true,
|
||||||
|
"Vue": true,
|
||||||
|
"SetVueGlobals": true,
|
||||||
|
"__": true,
|
||||||
|
"repl": true,
|
||||||
|
"Class": true,
|
||||||
|
"locals": true,
|
||||||
|
"cint": true,
|
||||||
|
"cstr": true,
|
||||||
|
"cur_frm": true,
|
||||||
|
"cur_dialog": true,
|
||||||
|
"cur_page": true,
|
||||||
|
"cur_list": true,
|
||||||
|
"cur_tree": true,
|
||||||
|
"msg_dialog": true,
|
||||||
|
"is_null": true,
|
||||||
|
"in_list": true,
|
||||||
|
"has_common": true,
|
||||||
|
"posthog": true,
|
||||||
|
"has_words": true,
|
||||||
|
"validate_email": true,
|
||||||
|
"open_web_template_values_editor": true,
|
||||||
|
"validate_name": true,
|
||||||
|
"validate_phone": true,
|
||||||
|
"validate_url": true,
|
||||||
|
"get_number_format": true,
|
||||||
|
"format_number": true,
|
||||||
|
"format_currency": true,
|
||||||
|
"comment_when": true,
|
||||||
|
"open_url_post": true,
|
||||||
|
"toTitle": true,
|
||||||
|
"lstrip": true,
|
||||||
|
"rstrip": true,
|
||||||
|
"strip": true,
|
||||||
|
"strip_html": true,
|
||||||
|
"replace_all": true,
|
||||||
|
"flt": true,
|
||||||
|
"precision": true,
|
||||||
|
"CREATE": true,
|
||||||
|
"AMEND": true,
|
||||||
|
"CANCEL": true,
|
||||||
|
"copy_dict": true,
|
||||||
|
"get_number_format_info": true,
|
||||||
|
"strip_number_groups": true,
|
||||||
|
"print_table": true,
|
||||||
|
"Layout": true,
|
||||||
|
"web_form_settings": true,
|
||||||
|
"$c": true,
|
||||||
|
"$a": true,
|
||||||
|
"$i": true,
|
||||||
|
"$bg": true,
|
||||||
|
"$y": true,
|
||||||
|
"$c_obj": true,
|
||||||
|
"refresh_many": true,
|
||||||
|
"refresh_field": true,
|
||||||
|
"toggle_field": true,
|
||||||
|
"get_field_obj": true,
|
||||||
|
"get_query_params": true,
|
||||||
|
"unhide_field": true,
|
||||||
|
"hide_field": true,
|
||||||
|
"set_field_options": true,
|
||||||
|
"getCookie": true,
|
||||||
|
"getCookies": true,
|
||||||
|
"get_url_arg": true,
|
||||||
|
"md5": true,
|
||||||
|
"$": true,
|
||||||
|
"jQuery": true,
|
||||||
|
"moment": true,
|
||||||
|
"hljs": true,
|
||||||
|
"Awesomplete": true,
|
||||||
|
"Sortable": true,
|
||||||
|
"Showdown": true,
|
||||||
|
"Taggle": true,
|
||||||
|
"Gantt": true,
|
||||||
|
"Slick": true,
|
||||||
|
"Webcam": true,
|
||||||
|
"PhotoSwipe": true,
|
||||||
|
"PhotoSwipeUI_Default": true,
|
||||||
|
"io": true,
|
||||||
|
"JsBarcode": true,
|
||||||
|
"L": true,
|
||||||
|
"Chart": true,
|
||||||
|
"DataTable": true,
|
||||||
|
"Cypress": true,
|
||||||
|
"cy": true,
|
||||||
|
"it": true,
|
||||||
|
"describe": true,
|
||||||
|
"expect": true,
|
||||||
|
"context": true,
|
||||||
|
"before": true,
|
||||||
|
"beforeEach": true,
|
||||||
|
"after": true,
|
||||||
|
"qz": true,
|
||||||
|
"localforage": true,
|
||||||
|
"extend_cscript": true
|
||||||
|
}
|
||||||
|
}
|
||||||
69
.pre-commit-config.yaml
Normal file
69
.pre-commit-config.yaml
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
exclude: 'node_modules|.git'
|
||||||
|
default_stages: [pre-commit]
|
||||||
|
fail_fast: false
|
||||||
|
|
||||||
|
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v5.0.0
|
||||||
|
hooks:
|
||||||
|
- id: trailing-whitespace
|
||||||
|
files: "sfda_parser.*"
|
||||||
|
exclude: ".*json$|.*txt$|.*csv|.*md|.*svg"
|
||||||
|
- id: check-merge-conflict
|
||||||
|
- id: check-ast
|
||||||
|
- id: check-json
|
||||||
|
- id: check-toml
|
||||||
|
- id: check-yaml
|
||||||
|
- id: debug-statements
|
||||||
|
|
||||||
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||||
|
rev: v0.8.1
|
||||||
|
hooks:
|
||||||
|
- id: ruff
|
||||||
|
name: "Run ruff import sorter"
|
||||||
|
args: ["--select=I", "--fix"]
|
||||||
|
|
||||||
|
- id: ruff
|
||||||
|
name: "Run ruff linter"
|
||||||
|
|
||||||
|
- id: ruff-format
|
||||||
|
name: "Run ruff formatter"
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||||
|
rev: v2.7.1
|
||||||
|
hooks:
|
||||||
|
- id: prettier
|
||||||
|
types_or: [javascript, vue, scss]
|
||||||
|
# Ignore any files that might contain jinja / bundles
|
||||||
|
exclude: |
|
||||||
|
(?x)^(
|
||||||
|
sfda_parser/public/dist/.*|
|
||||||
|
.*node_modules.*|
|
||||||
|
.*boilerplate.*|
|
||||||
|
sfda_parser/templates/includes/.*|
|
||||||
|
sfda_parser/public/js/lib/.*
|
||||||
|
)$
|
||||||
|
|
||||||
|
|
||||||
|
- repo: https://github.com/pre-commit/mirrors-eslint
|
||||||
|
rev: v8.44.0
|
||||||
|
hooks:
|
||||||
|
- id: eslint
|
||||||
|
types_or: [javascript]
|
||||||
|
args: ['--quiet']
|
||||||
|
# Ignore any files that might contain jinja / bundles
|
||||||
|
exclude: |
|
||||||
|
(?x)^(
|
||||||
|
sfda_parser/public/dist/.*|
|
||||||
|
cypress/.*|
|
||||||
|
.*node_modules.*|
|
||||||
|
.*boilerplate.*|
|
||||||
|
sfda_parser/templates/includes/.*|
|
||||||
|
sfda_parser/public/js/lib/.*
|
||||||
|
)$
|
||||||
|
|
||||||
|
ci:
|
||||||
|
autoupdate_schedule: weekly
|
||||||
|
skip: []
|
||||||
|
submodules: false
|
||||||
33
README.md
Normal file
33
README.md
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
### SFDA Parser
|
||||||
|
|
||||||
|
SFDA weekly alert scraper
|
||||||
|
|
||||||
|
### Installation
|
||||||
|
|
||||||
|
You can install this app using the [bench](https://github.com/frappe/bench) CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd $PATH_TO_YOUR_BENCH
|
||||||
|
bench get-app $URL_OF_THIS_REPO --branch develop
|
||||||
|
bench install-app sfda_parser
|
||||||
|
```
|
||||||
|
|
||||||
|
### Contributing
|
||||||
|
|
||||||
|
This app uses `pre-commit` for code formatting and linting. Please [install pre-commit](https://pre-commit.com/#installation) and enable it for this repository:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd apps/sfda_parser
|
||||||
|
pre-commit install
|
||||||
|
```
|
||||||
|
|
||||||
|
Pre-commit is configured to use the following tools for checking and formatting your code:
|
||||||
|
|
||||||
|
- ruff
|
||||||
|
- eslint
|
||||||
|
- prettier
|
||||||
|
- pyupgrade
|
||||||
|
|
||||||
|
### License
|
||||||
|
|
||||||
|
mit
|
||||||
21
license.txt
Normal file
21
license.txt
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) [year] [fullname]
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
59
pyproject.toml
Normal file
59
pyproject.toml
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
[project]
|
||||||
|
name = "sfda_parser"
|
||||||
|
authors = [
|
||||||
|
{ name = "seyfert", email = "support@seeraarabia.com"}
|
||||||
|
]
|
||||||
|
description = "SFDA weekly alert scraper"
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
readme = "README.md"
|
||||||
|
dynamic = ["version"]
|
||||||
|
dependencies = [
|
||||||
|
# "frappe~=15.0.0" # Installed and managed by bench.
|
||||||
|
"beautifulsoup4>=4.12.0",
|
||||||
|
"pdfplumber>=0.11.0",
|
||||||
|
"requests>=2.31.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["flit_core >=3.4,<4"]
|
||||||
|
build-backend = "flit_core.buildapi"
|
||||||
|
|
||||||
|
# These dependencies are only installed when developer mode is enabled
|
||||||
|
[tool.bench.dev-dependencies]
|
||||||
|
# package_name = "~=1.1.0"
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 110
|
||||||
|
target-version = "py310"
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = [
|
||||||
|
"F",
|
||||||
|
"E",
|
||||||
|
"W",
|
||||||
|
"I",
|
||||||
|
"UP",
|
||||||
|
"B",
|
||||||
|
"RUF",
|
||||||
|
]
|
||||||
|
ignore = [
|
||||||
|
"B017", # assertRaises(Exception) - should be more specific
|
||||||
|
"B018", # useless expression, not assigned to anything
|
||||||
|
"B023", # function doesn't bind loop variable - will have last iteration's value
|
||||||
|
"B904", # raise inside except without from
|
||||||
|
"E101", # indentation contains mixed spaces and tabs
|
||||||
|
"E402", # module level import not at top of file
|
||||||
|
"E501", # line too long
|
||||||
|
"E741", # ambiguous variable name
|
||||||
|
"F401", # "unused" imports
|
||||||
|
"F403", # can't detect undefined names from * import
|
||||||
|
"F405", # can't detect undefined names from * import
|
||||||
|
"F722", # syntax error in forward type annotation
|
||||||
|
"W191", # indentation contains tabs
|
||||||
|
]
|
||||||
|
typing-modules = ["frappe.types.DF"]
|
||||||
|
|
||||||
|
[tool.ruff.format]
|
||||||
|
quote-style = "double"
|
||||||
|
indent-style = "tab"
|
||||||
|
docstring-code-format = true
|
||||||
1
sfda_parser/__init__.py
Normal file
1
sfda_parser/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
__version__ = "0.0.1"
|
||||||
BIN
sfda_parser/__pycache__/__init__.cpython-310.pyc
Normal file
BIN
sfda_parser/__pycache__/__init__.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/__pycache__/hooks.cpython-310.pyc
Normal file
BIN
sfda_parser/__pycache__/hooks.cpython-310.pyc
Normal file
Binary file not shown.
0
sfda_parser/api/__init__.py
Normal file
0
sfda_parser/api/__init__.py
Normal file
BIN
sfda_parser/api/__pycache__/__init__.cpython-310.pyc
Normal file
BIN
sfda_parser/api/__pycache__/__init__.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/api/__pycache__/sfda_scraper.cpython-310.pyc
Normal file
BIN
sfda_parser/api/__pycache__/sfda_scraper.cpython-310.pyc
Normal file
Binary file not shown.
87
sfda_parser/api/sfda_scraper.py
Normal file
87
sfda_parser/api/sfda_scraper.py
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
"""Whitelisted API for SFDA scraper and ADE PublishDetails preview."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_page import (
|
||||||
|
fetch_publish_details,
|
||||||
|
resolve_detail_url_for_ncmdr_ref,
|
||||||
|
)
|
||||||
|
from sfda_parser.sfda_scraper.importer import import_latest_weekly_alerts
|
||||||
|
from sfda_parser.sfda_scraper.reparse import reparse_all_stored_pdf_device_lists
|
||||||
|
|
||||||
|
|
||||||
|
@frappe.whitelist()
|
||||||
|
def run_sfda_scraper_now():
|
||||||
|
frappe.only_for("System Manager")
|
||||||
|
return import_latest_weekly_alerts()
|
||||||
|
|
||||||
|
|
||||||
|
@frappe.whitelist()
|
||||||
|
def reparse_device_list_from_stored_pdfs(sfda_entries_name: str | None = None):
|
||||||
|
"""Rebuild Device List child rows from stored detail_pdf using current PDF parser."""
|
||||||
|
frappe.only_for("System Manager")
|
||||||
|
return reparse_all_stored_pdf_device_lists(sfda_entries_name=sfda_entries_name)
|
||||||
|
|
||||||
|
|
||||||
|
@frappe.whitelist()
|
||||||
|
def get_ade_publish_details(
|
||||||
|
detail_url: str | None = None,
|
||||||
|
sfda_entries_name: str | None = None,
|
||||||
|
sfda_device_entry_name: str | None = None,
|
||||||
|
ncmdr_ref: str | None = None,
|
||||||
|
include_device_list_from_pdf: int | str = 0,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Load ADE PublishDetails via the ERP server (works when your PC browser is blocked).
|
||||||
|
Pass detail_url and/or sfda_entries_name with a child row reference.
|
||||||
|
"""
|
||||||
|
include_pdf = str(include_device_list_from_pdf).lower() in ("1", "true", "yes")
|
||||||
|
|
||||||
|
url = (detail_url or "").strip()
|
||||||
|
if sfda_entries_name and not url:
|
||||||
|
doc = frappe.get_doc("SFDA Entries", sfda_entries_name)
|
||||||
|
child_row = None
|
||||||
|
|
||||||
|
if sfda_device_entry_name:
|
||||||
|
child_row = next(
|
||||||
|
(row for row in doc.device_list if row.name == sfda_device_entry_name),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
elif ncmdr_ref:
|
||||||
|
target = (ncmdr_ref or "").strip().upper()
|
||||||
|
child_row = next(
|
||||||
|
(
|
||||||
|
row
|
||||||
|
for row in doc.device_list
|
||||||
|
if (row.ncmdr_ref or "").strip().upper() == target
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if child_row:
|
||||||
|
url = (child_row.get("ade_detail_url") or "").strip()
|
||||||
|
if not url and child_row.ncmdr_ref:
|
||||||
|
url = resolve_detail_url_for_ncmdr_ref(child_row.ncmdr_ref) or ""
|
||||||
|
if url:
|
||||||
|
child_row.ade_detail_url = url
|
||||||
|
doc.save(ignore_permissions=True)
|
||||||
|
|
||||||
|
if not url:
|
||||||
|
frappe.throw(
|
||||||
|
"No ADE URL available. Pass detail_url, select a device row with an ADE URL, "
|
||||||
|
"or re-run the weekly import."
|
||||||
|
)
|
||||||
|
|
||||||
|
if not url.startswith("http"):
|
||||||
|
url = f"https://ade.sfda.gov.sa/Fsca/PublishDetails/{url}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
return fetch_publish_details(url, include_device_list_from_pdf=include_pdf)
|
||||||
|
except Exception as exc:
|
||||||
|
frappe.log_error(title="SFDA: ADE fetch failed", message=f"URL: {url}\n{exc}")
|
||||||
|
frappe.throw(
|
||||||
|
f"Could not load ADE from the server: {exc}. "
|
||||||
|
"The site may be blocked on your PC but reachable from ERP."
|
||||||
|
)
|
||||||
0
sfda_parser/config/__init__.py
Normal file
0
sfda_parser/config/__init__.py
Normal file
1
sfda_parser/fixtures/client_script.json
Normal file
1
sfda_parser/fixtures/client_script.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
15722
sfda_parser/fixtures/custom_docperm.json
Normal file
15722
sfda_parser/fixtures/custom_docperm.json
Normal file
File diff suppressed because it is too large
Load Diff
59
sfda_parser/fixtures/custom_field.json
Normal file
59
sfda_parser/fixtures/custom_field.json
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"allow_in_quick_entry": 0,
|
||||||
|
"allow_on_submit": 1,
|
||||||
|
"bold": 0,
|
||||||
|
"collapsible": 0,
|
||||||
|
"collapsible_depends_on": null,
|
||||||
|
"columns": 0,
|
||||||
|
"default": null,
|
||||||
|
"depends_on": null,
|
||||||
|
"description": null,
|
||||||
|
"docstatus": 0,
|
||||||
|
"doctype": "Custom Field",
|
||||||
|
"dt": "Asset",
|
||||||
|
"fetch_from": null,
|
||||||
|
"fetch_if_empty": 0,
|
||||||
|
"fieldname": "custom_recalled",
|
||||||
|
"fieldtype": "Select",
|
||||||
|
"hidden": 0,
|
||||||
|
"hide_border": 0,
|
||||||
|
"hide_days": 0,
|
||||||
|
"hide_seconds": 0,
|
||||||
|
"ignore_user_permissions": 0,
|
||||||
|
"ignore_xss_filter": 0,
|
||||||
|
"in_global_search": 0,
|
||||||
|
"in_list_view": 0,
|
||||||
|
"in_preview": 0,
|
||||||
|
"in_standard_filter": 0,
|
||||||
|
"insert_after": "custom_category",
|
||||||
|
"is_system_generated": 0,
|
||||||
|
"is_virtual": 0,
|
||||||
|
"label": "Recalled",
|
||||||
|
"length": 0,
|
||||||
|
"link_filters": null,
|
||||||
|
"mandatory_depends_on": null,
|
||||||
|
"modified": "2026-06-05 12:26:39.091744",
|
||||||
|
"module": "SFDA Parser",
|
||||||
|
"name": "Asset-custom_recalled",
|
||||||
|
"no_copy": 0,
|
||||||
|
"non_negative": 0,
|
||||||
|
"options": "\nYes\nNo",
|
||||||
|
"permlevel": 0,
|
||||||
|
"placeholder": null,
|
||||||
|
"precision": "",
|
||||||
|
"print_hide": 0,
|
||||||
|
"print_hide_if_no_value": 0,
|
||||||
|
"print_width": null,
|
||||||
|
"read_only": 0,
|
||||||
|
"read_only_depends_on": null,
|
||||||
|
"report_hide": 0,
|
||||||
|
"reqd": 0,
|
||||||
|
"search_index": 0,
|
||||||
|
"show_dashboard": 0,
|
||||||
|
"sort_options": 0,
|
||||||
|
"translatable": 0,
|
||||||
|
"unique": 0,
|
||||||
|
"width": null
|
||||||
|
}
|
||||||
|
]
|
||||||
1
sfda_parser/fixtures/property_setter.json
Normal file
1
sfda_parser/fixtures/property_setter.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
1
sfda_parser/fixtures/report.json
Normal file
1
sfda_parser/fixtures/report.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
[]
|
||||||
21
sfda_parser/fixtures/server_script.json
Normal file
21
sfda_parser/fixtures/server_script.json
Normal file
File diff suppressed because one or more lines are too long
262
sfda_parser/hooks.py
Normal file
262
sfda_parser/hooks.py
Normal file
@ -0,0 +1,262 @@
|
|||||||
|
app_name = "sfda_parser"
|
||||||
|
app_title = "SFDA Parser"
|
||||||
|
app_publisher = "seyfert"
|
||||||
|
app_description = "SFDA weekly alert scraper"
|
||||||
|
app_email = "support@seeraarabia.com"
|
||||||
|
app_license = "mit"
|
||||||
|
|
||||||
|
# Apps
|
||||||
|
# ------------------
|
||||||
|
|
||||||
|
# required_apps = []
|
||||||
|
|
||||||
|
# Each item in the list will be shown as an app in the apps page
|
||||||
|
# add_to_apps_screen = [
|
||||||
|
# {
|
||||||
|
# "name": "sfda_parser",
|
||||||
|
# "logo": "/assets/sfda_parser/logo.png",
|
||||||
|
# "title": "SFDA Parser",
|
||||||
|
# "route": "/sfda_parser",
|
||||||
|
# "has_permission": "sfda_parser.api.permission.has_app_permission"
|
||||||
|
# }
|
||||||
|
# ]
|
||||||
|
|
||||||
|
# Includes in <head>
|
||||||
|
# ------------------
|
||||||
|
|
||||||
|
# include js, css files in header of desk.html
|
||||||
|
# app_include_css = "/assets/sfda_parser/css/sfda_parser.css"
|
||||||
|
# app_include_js = "/assets/sfda_parser/js/sfda_parser.js"
|
||||||
|
|
||||||
|
# include js, css files in header of web template
|
||||||
|
# web_include_css = "/assets/sfda_parser/css/sfda_parser.css"
|
||||||
|
# web_include_js = "/assets/sfda_parser/js/sfda_parser.js"
|
||||||
|
|
||||||
|
# include custom scss in every website theme (without file extension ".scss")
|
||||||
|
# website_theme_scss = "sfda_parser/public/scss/website"
|
||||||
|
|
||||||
|
# include js, css files in header of web form
|
||||||
|
# webform_include_js = {"doctype": "public/js/doctype.js"}
|
||||||
|
# webform_include_css = {"doctype": "public/css/doctype.css"}
|
||||||
|
|
||||||
|
# include js in page
|
||||||
|
# page_js = {"page" : "public/js/file.js"}
|
||||||
|
|
||||||
|
# include js in doctype views
|
||||||
|
# doctype_js = {"doctype" : "public/js/doctype.js"}
|
||||||
|
# doctype_list_js = {"doctype" : "public/js/doctype_list.js"}
|
||||||
|
# doctype_tree_js = {"doctype" : "public/js/doctype_tree.js"}
|
||||||
|
# doctype_calendar_js = {"doctype" : "public/js/doctype_calendar.js"}
|
||||||
|
|
||||||
|
# Svg Icons
|
||||||
|
# ------------------
|
||||||
|
# include app icons in desk
|
||||||
|
# app_include_icons = "sfda_parser/public/icons.svg"
|
||||||
|
|
||||||
|
# Home Pages
|
||||||
|
# ----------
|
||||||
|
|
||||||
|
# application home page (will override Website Settings)
|
||||||
|
# home_page = "login"
|
||||||
|
|
||||||
|
# website user home page (by Role)
|
||||||
|
# role_home_page = {
|
||||||
|
# "Role": "home_page"
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Generators
|
||||||
|
# ----------
|
||||||
|
|
||||||
|
# automatically create page for each record of this doctype
|
||||||
|
# website_generators = ["Web Page"]
|
||||||
|
|
||||||
|
# Jinja
|
||||||
|
# ----------
|
||||||
|
|
||||||
|
# add methods and filters to jinja environment
|
||||||
|
# jinja = {
|
||||||
|
# "methods": "sfda_parser.utils.jinja_methods",
|
||||||
|
# "filters": "sfda_parser.utils.jinja_filters"
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Installation
|
||||||
|
# ------------
|
||||||
|
|
||||||
|
# before_install = "sfda_parser.install.before_install"
|
||||||
|
# after_install = "sfda_parser.install.after_install"
|
||||||
|
|
||||||
|
# Uninstallation
|
||||||
|
# ------------
|
||||||
|
|
||||||
|
# before_uninstall = "sfda_parser.uninstall.before_uninstall"
|
||||||
|
# after_uninstall = "sfda_parser.uninstall.after_uninstall"
|
||||||
|
|
||||||
|
# Integration Setup
|
||||||
|
# ------------------
|
||||||
|
# To set up dependencies/integrations with other apps
|
||||||
|
# Name of the app being installed is passed as an argument
|
||||||
|
|
||||||
|
# before_app_install = "sfda_parser.utils.before_app_install"
|
||||||
|
# after_app_install = "sfda_parser.utils.after_app_install"
|
||||||
|
|
||||||
|
# Integration Cleanup
|
||||||
|
# -------------------
|
||||||
|
# To clean up dependencies/integrations with other apps
|
||||||
|
# Name of the app being uninstalled is passed as an argument
|
||||||
|
|
||||||
|
# before_app_uninstall = "sfda_parser.utils.before_app_uninstall"
|
||||||
|
# after_app_uninstall = "sfda_parser.utils.after_app_uninstall"
|
||||||
|
|
||||||
|
# Desk Notifications
|
||||||
|
# ------------------
|
||||||
|
# See frappe.core.notifications.get_notification_config
|
||||||
|
|
||||||
|
# notification_config = "sfda_parser.notifications.get_notification_config"
|
||||||
|
|
||||||
|
# Permissions
|
||||||
|
# -----------
|
||||||
|
# Permissions evaluated in scripted ways
|
||||||
|
|
||||||
|
# permission_query_conditions = {
|
||||||
|
# "Event": "frappe.desk.doctype.event.event.get_permission_query_conditions",
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# has_permission = {
|
||||||
|
# "Event": "frappe.desk.doctype.event.event.has_permission",
|
||||||
|
# }
|
||||||
|
|
||||||
|
# DocType Class
|
||||||
|
# ---------------
|
||||||
|
# Override standard doctype classes
|
||||||
|
|
||||||
|
# override_doctype_class = {
|
||||||
|
# "ToDo": "custom_app.overrides.CustomToDo"
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Document Events
|
||||||
|
# ---------------
|
||||||
|
# Hook on document methods and events
|
||||||
|
|
||||||
|
# doc_events = {
|
||||||
|
# "*": {
|
||||||
|
# "on_update": "method",
|
||||||
|
# "on_cancel": "method",
|
||||||
|
# "on_trash": "method"
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Scheduled Tasks
|
||||||
|
# ---------------
|
||||||
|
|
||||||
|
# SFDA recall scraper: every Monday 07:00 (server timezone).
|
||||||
|
# Always imports from the newest row on https://www.sfda.gov.sa/en/weekly-alert only.
|
||||||
|
scheduler_events = {
|
||||||
|
"cron": {
|
||||||
|
"0 7 * * 1": [
|
||||||
|
"sfda_parser.sfda_scraper.jobs.run_weekly_sfda_import_scheduled",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Testing
|
||||||
|
# -------
|
||||||
|
|
||||||
|
# before_tests = "sfda_parser.install.before_tests"
|
||||||
|
|
||||||
|
# Overriding Methods
|
||||||
|
# ------------------------------
|
||||||
|
#
|
||||||
|
# override_whitelisted_methods = {
|
||||||
|
# "frappe.desk.doctype.event.event.get_events": "sfda_parser.event.get_events"
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# each overriding function accepts a `data` argument;
|
||||||
|
# generated from the base implementation of the doctype dashboard,
|
||||||
|
# along with any modifications made in other Frappe apps
|
||||||
|
# override_doctype_dashboards = {
|
||||||
|
# "Task": "sfda_parser.task.get_dashboard_data"
|
||||||
|
# }
|
||||||
|
|
||||||
|
# exempt linked doctypes from being automatically cancelled
|
||||||
|
#
|
||||||
|
# auto_cancel_exempted_doctypes = ["Auto Repeat"]
|
||||||
|
|
||||||
|
# Ignore links to specified DocTypes when deleting documents
|
||||||
|
# -----------------------------------------------------------
|
||||||
|
|
||||||
|
# ignore_links_on_delete = ["Communication", "ToDo"]
|
||||||
|
|
||||||
|
# Request Events
|
||||||
|
# ----------------
|
||||||
|
# before_request = ["sfda_parser.utils.before_request"]
|
||||||
|
# after_request = ["sfda_parser.utils.after_request"]
|
||||||
|
|
||||||
|
# Job Events
|
||||||
|
# ----------
|
||||||
|
# before_job = ["sfda_parser.utils.before_job"]
|
||||||
|
# after_job = ["sfda_parser.utils.after_job"]
|
||||||
|
|
||||||
|
# User Data Protection
|
||||||
|
# --------------------
|
||||||
|
|
||||||
|
# user_data_fields = [
|
||||||
|
# {
|
||||||
|
# "doctype": "{doctype_1}",
|
||||||
|
# "filter_by": "{filter_by}",
|
||||||
|
# "redact_fields": ["{field_1}", "{field_2}"],
|
||||||
|
# "partial": 1,
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "doctype": "{doctype_2}",
|
||||||
|
# "filter_by": "{filter_by}",
|
||||||
|
# "partial": 1,
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "doctype": "{doctype_3}",
|
||||||
|
# "strict": False,
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "doctype": "{doctype_4}"
|
||||||
|
# }
|
||||||
|
# ]
|
||||||
|
|
||||||
|
# Authentication and authorization
|
||||||
|
# --------------------------------
|
||||||
|
|
||||||
|
# auth_hooks = [
|
||||||
|
# "sfda_parser.auth.validate"
|
||||||
|
# ]
|
||||||
|
|
||||||
|
# Automatically update python controller files with type annotations for this app.
|
||||||
|
# export_python_type_annotations = True
|
||||||
|
|
||||||
|
# default_log_clearing_doctypes = {
|
||||||
|
# "Logging DocType Name": 30 # days to retain logs
|
||||||
|
# }
|
||||||
|
|
||||||
|
# Translation
|
||||||
|
# ------------
|
||||||
|
# List of apps whose translatable strings should be excluded from this app's translations.
|
||||||
|
# ignore_translatable_strings_from = []
|
||||||
|
|
||||||
|
fixtures = [
|
||||||
|
"Custom DocPerm",
|
||||||
|
|
||||||
|
{"doctype": "Custom Field", "filters": [
|
||||||
|
[
|
||||||
|
"module", "=", "SFDA Parser"
|
||||||
|
]
|
||||||
|
]},
|
||||||
|
{"doctype": "Property Setter", "filters": [
|
||||||
|
[
|
||||||
|
"module", "=", "SFDA Parser"
|
||||||
|
]
|
||||||
|
]},
|
||||||
|
{"dt": "Print Format", "filters": {"custom_format": 1}},
|
||||||
|
{"doctype": "Client Script", "filters": [["module","=","SFDA Parser"]]},
|
||||||
|
{"doctype": "Server Script", "filters": [["module","=","SFDA Parser"]]},
|
||||||
|
{"doctype": "Report", "filters": [["module","=","SFDA Parser"]]}
|
||||||
|
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
1
sfda_parser/modules.txt
Normal file
1
sfda_parser/modules.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
SFDA Parser
|
||||||
7
sfda_parser/patches.txt
Normal file
7
sfda_parser/patches.txt
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
[pre_model_sync]
|
||||||
|
# Patches added in this section will be executed before doctypes are migrated
|
||||||
|
# Read docs to understand patches: https://frappeframework.com/docs/v14/user/en/database-migrations
|
||||||
|
|
||||||
|
[post_model_sync]
|
||||||
|
# Patches added in this section will be executed after doctypes are migrated
|
||||||
|
sfda_parser.patches.reparse_sfda_device_lists_from_pdfs
|
||||||
0
sfda_parser/patches/__init__.py
Normal file
0
sfda_parser/patches/__init__.py
Normal file
BIN
sfda_parser/patches/__pycache__/__init__.cpython-310.pyc
Normal file
BIN
sfda_parser/patches/__pycache__/__init__.cpython-310.pyc
Normal file
Binary file not shown.
Binary file not shown.
20
sfda_parser/patches/reparse_sfda_device_lists_from_pdfs.py
Normal file
20
sfda_parser/patches/reparse_sfda_device_lists_from_pdfs.py
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
"""One-time post-migrate: refresh device_list from stored detail PDFs after header alias updates."""
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.reparse import reparse_all_stored_pdf_device_lists
|
||||||
|
|
||||||
|
|
||||||
|
def execute():
|
||||||
|
if not frappe.db.exists("DocType", "SFDA Entries"):
|
||||||
|
return
|
||||||
|
|
||||||
|
summary = reparse_all_stored_pdf_device_lists()
|
||||||
|
frappe.logger("sfda_parser").info(
|
||||||
|
"Re-parsed SFDA device lists from PDFs: %s", summary
|
||||||
|
)
|
||||||
|
if summary.get("errors"):
|
||||||
|
frappe.log_error(
|
||||||
|
title="SFDA Parser: PDF reparse completed with errors",
|
||||||
|
message=frappe.as_json(summary, indent=2),
|
||||||
|
)
|
||||||
0
sfda_parser/public/.gitkeep
Normal file
0
sfda_parser/public/.gitkeep
Normal file
0
sfda_parser/sfda_parser/__init__.py
Normal file
0
sfda_parser/sfda_parser/__init__.py
Normal file
BIN
sfda_parser/sfda_parser/__pycache__/__init__.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_parser/__pycache__/__init__.cpython-310.pyc
Normal file
Binary file not shown.
0
sfda_parser/sfda_parser/doctype/__init__.py
Normal file
0
sfda_parser/sfda_parser/doctype/__init__.py
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,120 @@
|
|||||||
|
{
|
||||||
|
"actions": [],
|
||||||
|
"allow_rename": 1,
|
||||||
|
"creation": "2026-06-01 00:00:00",
|
||||||
|
"doctype": "DocType",
|
||||||
|
"editable_grid": 1,
|
||||||
|
"engine": "InnoDB",
|
||||||
|
"field_order": [
|
||||||
|
"ncmdr_ref",
|
||||||
|
"manufacturer",
|
||||||
|
"detail_pdf",
|
||||||
|
"ade_detail_url",
|
||||||
|
"column_break_alert",
|
||||||
|
"material",
|
||||||
|
"material_description",
|
||||||
|
"catalog_number",
|
||||||
|
"udi",
|
||||||
|
"serial_no",
|
||||||
|
"gstn",
|
||||||
|
"batch",
|
||||||
|
"serial_no_matching"
|
||||||
|
],
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"fieldname": "ncmdr_ref",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"in_standard_filter": 1,
|
||||||
|
"label": "NCMDR Ref"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "manufacturer",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"in_standard_filter": 1,
|
||||||
|
"label": "Manufacturer",
|
||||||
|
"read_only": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "detail_pdf",
|
||||||
|
"fieldtype": "Attach",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "ADE Detail PDF",
|
||||||
|
"read_only": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "ade_detail_url",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"hidden": 1,
|
||||||
|
"label": "ADE Detail URL",
|
||||||
|
"read_only": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "column_break_alert",
|
||||||
|
"fieldtype": "Column Break"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "material",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "Material"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "material_description",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "Material Description"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "catalog_number",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "Catalog Number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "udi",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "UDI"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "serial_no",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "Serial No"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "gstn",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"label": "GTIN"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "batch",
|
||||||
|
"fieldtype": "Small Text",
|
||||||
|
"label": "Batch"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "serial_no_matching",
|
||||||
|
"fieldtype": "Select",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"label": "Matching",
|
||||||
|
"options": "\nYes\nNo"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"grid_page_length": 50,
|
||||||
|
"index_web_pages_for_search": 1,
|
||||||
|
"istable": 1,
|
||||||
|
"links": [],
|
||||||
|
"modified": "2026-06-09 13:37:08.969913",
|
||||||
|
"modified_by": "support@seeraarabia.com",
|
||||||
|
"module": "SFDA Parser",
|
||||||
|
"name": "SFDA Device Entries",
|
||||||
|
"owner": "Administrator",
|
||||||
|
"permissions": [],
|
||||||
|
"row_format": "Dynamic",
|
||||||
|
"rows_threshold_for_grid_search": 20,
|
||||||
|
"sort_field": "modified",
|
||||||
|
"sort_order": "DESC",
|
||||||
|
"states": []
|
||||||
|
}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# Copyright (c) 2026, seyfert and contributors
|
||||||
|
# For license information, please see license.txt
|
||||||
|
|
||||||
|
from frappe.model.document import Document
|
||||||
|
|
||||||
|
|
||||||
|
class SFDADeviceEntries(Document):
|
||||||
|
pass
|
||||||
Binary file not shown.
Binary file not shown.
199
sfda_parser/sfda_parser/doctype/sfda_entries/sfda_entries.js
Normal file
199
sfda_parser/sfda_parser/doctype/sfda_entries/sfda_entries.js
Normal file
@ -0,0 +1,199 @@
|
|||||||
|
// Copyright (c) 2026, seyfert and contributors
|
||||||
|
|
||||||
|
const ADE_FIELD_LABELS = {
|
||||||
|
reference_number: "NCMDR / Reference",
|
||||||
|
manufacturer: "Manufacturer",
|
||||||
|
product_trade_name: "Product / Trade Name",
|
||||||
|
model: "Model",
|
||||||
|
authorized_representative: "Authorized Representative",
|
||||||
|
problem_reason: "Problem / Reason",
|
||||||
|
action: "Corrective Action",
|
||||||
|
};
|
||||||
|
|
||||||
|
function format_ade_value(value) {
|
||||||
|
if (!value) {
|
||||||
|
return "—";
|
||||||
|
}
|
||||||
|
return frappe.utils.escape_html(String(value)).replace(/\n/g, "<br>");
|
||||||
|
}
|
||||||
|
|
||||||
|
function build_ade_dialog_html(frm, data, child_row) {
|
||||||
|
let rows = "";
|
||||||
|
for (const [key, label] of Object.entries(ADE_FIELD_LABELS)) {
|
||||||
|
rows += `<tr>
|
||||||
|
<td class="text-muted" style="width:35%;vertical-align:top;"><b>${label}</b></td>
|
||||||
|
<td>${format_ade_value(data[key])}</td>
|
||||||
|
</tr>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const erp_count = (frm.doc.device_list || []).filter(
|
||||||
|
(row) => row.ncmdr_ref === child_row.ncmdr_ref
|
||||||
|
).length;
|
||||||
|
const pdf_count = data.device_list_from_pdf_count || 0;
|
||||||
|
|
||||||
|
let device_section = `<hr><p><b>Device rows for ${format_ade_value(
|
||||||
|
child_row.ncmdr_ref
|
||||||
|
)} in ERP:</b> ${erp_count} row(s)</p>`;
|
||||||
|
device_section += `<p><b>Device rows parsed from PDF (live):</b> ${pdf_count}</p>`;
|
||||||
|
if (data.pdf_parse_error) {
|
||||||
|
device_section += `<p class="text-danger"><b>PDF note:</b> ${format_ade_value(
|
||||||
|
data.pdf_parse_error
|
||||||
|
)}</p>`;
|
||||||
|
}
|
||||||
|
if (pdf_count > 0 && data.device_list_from_pdf) {
|
||||||
|
device_section += `<table class="table table-bordered table-sm"><thead><tr>
|
||||||
|
<th>Material</th><th>Description</th><th>Catalog No</th><th>UDI</th><th>Serial No</th><th>GTIN</th><th>Batch</th>
|
||||||
|
</tr></thead><tbody>`;
|
||||||
|
data.device_list_from_pdf.forEach((row) => {
|
||||||
|
device_section += `<tr>
|
||||||
|
<td>${format_ade_value(row.material)}</td>
|
||||||
|
<td>${format_ade_value(row.material_description)}</td>
|
||||||
|
<td>${format_ade_value(row.catalog_number)}</td>
|
||||||
|
<td>${format_ade_value(row.udi)}</td>
|
||||||
|
<td>${format_ade_value(row.serial_no)}</td>
|
||||||
|
<td>${format_ade_value(row.gstn)}</td>
|
||||||
|
<td>${format_ade_value(row.batch)}</td>
|
||||||
|
</tr>`;
|
||||||
|
});
|
||||||
|
device_section += "</tbody></table>";
|
||||||
|
}
|
||||||
|
|
||||||
|
const url_line = data.detail_url
|
||||||
|
? `<p class="text-muted small">Source: ${format_ade_value(data.detail_url)}</p>`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
const pdf_link = child_row.detail_pdf
|
||||||
|
? `<p><a href="${frappe.utils.escape_html(child_row.detail_pdf)}" target="_blank" rel="noopener">
|
||||||
|
${__("Download original PDF")}</a></p>`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
return `<div style="max-height:65vh;overflow:auto;">
|
||||||
|
<p class="text-muted small">Loaded via ERP server — ADE may be blocked in your local browser.</p>
|
||||||
|
${pdf_link}
|
||||||
|
${url_line}
|
||||||
|
<table class="table table-bordered">${rows}</table>
|
||||||
|
${device_section}
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MATCHING_YES_ROW_COLOR = "#ffedd5";
|
||||||
|
|
||||||
|
function highlight_matching_device_rows(frm) {
|
||||||
|
const grid = frm.fields_dict.device_list?.grid;
|
||||||
|
if (!grid?.grid_rows) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
grid.grid_rows.forEach((grid_row) => {
|
||||||
|
const is_match =
|
||||||
|
String(grid_row.doc.serial_no_matching || "").trim() === "Yes";
|
||||||
|
$(grid_row.row).css(
|
||||||
|
"background-color",
|
||||||
|
is_match ? MATCHING_YES_ROW_COLOR : ""
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_selected_device_row(frm) {
|
||||||
|
const grid = frm.fields_dict.device_list?.grid;
|
||||||
|
if (!grid) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const selected = grid.get_selected_children();
|
||||||
|
if (selected?.length === 1) {
|
||||||
|
return selected[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (grid.grid_rows?.length === 1) {
|
||||||
|
return grid.grid_rows[0].doc;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
frappe.ui.form.on("SFDA Entries", {
|
||||||
|
refresh(frm) {
|
||||||
|
highlight_matching_device_rows(frm);
|
||||||
|
|
||||||
|
frm.add_custom_button(
|
||||||
|
__("View ADE details"),
|
||||||
|
() => {
|
||||||
|
const child_row = get_selected_device_row(frm);
|
||||||
|
if (!child_row) {
|
||||||
|
frappe.msgprint(
|
||||||
|
__("Please select exactly one row in Device List to view ADE details.")
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
frappe.call({
|
||||||
|
method: "sfda_parser.api.sfda_scraper.get_ade_publish_details",
|
||||||
|
args: {
|
||||||
|
sfda_entries_name: frm.doc.name,
|
||||||
|
sfda_device_entry_name: child_row.name,
|
||||||
|
include_device_list_from_pdf: 1,
|
||||||
|
},
|
||||||
|
freeze: true,
|
||||||
|
freeze_message: __("Loading from ADE (server)..."),
|
||||||
|
callback(r) {
|
||||||
|
if (!r.message) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const d = new frappe.ui.Dialog({
|
||||||
|
title: __("ADE Publish Details"),
|
||||||
|
size: "large",
|
||||||
|
fields: [{ fieldtype: "HTML", fieldname: "body" }],
|
||||||
|
primary_action_label: __("Close"),
|
||||||
|
primary_action() {
|
||||||
|
d.hide();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
d.fields_dict.body.$wrapper.html(
|
||||||
|
build_ade_dialog_html(frm, r.message, child_row)
|
||||||
|
);
|
||||||
|
d.show();
|
||||||
|
|
||||||
|
const needs_save =
|
||||||
|
(r.message.detail_url && !child_row.ade_detail_url) ||
|
||||||
|
(r.message.manufacturer && !child_row.manufacturer);
|
||||||
|
if (r.message.detail_url && !child_row.ade_detail_url) {
|
||||||
|
frappe.model.set_value(
|
||||||
|
child_row.doctype,
|
||||||
|
child_row.name,
|
||||||
|
"ade_detail_url",
|
||||||
|
r.message.detail_url
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (r.message.manufacturer && !child_row.manufacturer) {
|
||||||
|
frappe.model.set_value(
|
||||||
|
child_row.doctype,
|
||||||
|
child_row.name,
|
||||||
|
"manufacturer",
|
||||||
|
r.message.manufacturer
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (needs_save) {
|
||||||
|
frm.save();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
__("Actions")
|
||||||
|
);
|
||||||
|
},
|
||||||
|
device_list_on_form_rendered(frm) {
|
||||||
|
highlight_matching_device_rows(frm);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
frappe.ui.form.on("SFDA Device Entries", {
|
||||||
|
serial_no_matching(frm, cdt, cdn) {
|
||||||
|
const grid_row = frm.fields_dict.device_list?.grid?.grid_rows_by_docname?.[cdn];
|
||||||
|
if (!grid_row) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const is_match = String(frappe.model.get_value(cdt, cdn, "serial_no_matching") || "").trim() === "Yes";
|
||||||
|
$(grid_row.row).css("background-color", is_match ? MATCHING_YES_ROW_COLOR : "");
|
||||||
|
},
|
||||||
|
});
|
||||||
@ -0,0 +1,98 @@
|
|||||||
|
{
|
||||||
|
"actions": [],
|
||||||
|
"allow_rename": 1,
|
||||||
|
"autoname": "field:title",
|
||||||
|
"creation": "2026-06-01 16:16:53.409449",
|
||||||
|
"doctype": "DocType",
|
||||||
|
"engine": "InnoDB",
|
||||||
|
"field_order": [
|
||||||
|
"title",
|
||||||
|
"column_break_dqny",
|
||||||
|
"date",
|
||||||
|
"section_break_qcgk",
|
||||||
|
"device_list",
|
||||||
|
"section_break_ihhu",
|
||||||
|
"passed_date",
|
||||||
|
"column_break_whlv",
|
||||||
|
"passed"
|
||||||
|
],
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"fieldname": "title",
|
||||||
|
"fieldtype": "Data",
|
||||||
|
"in_global_search": 1,
|
||||||
|
"in_list_view": 1,
|
||||||
|
"in_standard_filter": 1,
|
||||||
|
"label": "Title",
|
||||||
|
"unique": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "column_break_dqny",
|
||||||
|
"fieldtype": "Column Break"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "date",
|
||||||
|
"fieldtype": "Date",
|
||||||
|
"in_list_view": 1,
|
||||||
|
"in_standard_filter": 1,
|
||||||
|
"label": "Date"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "section_break_qcgk",
|
||||||
|
"fieldtype": "Section Break"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "device_list",
|
||||||
|
"fieldtype": "Table",
|
||||||
|
"label": "Device List",
|
||||||
|
"options": "SFDA Device Entries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "section_break_ihhu",
|
||||||
|
"fieldtype": "Section Break"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "passed_date",
|
||||||
|
"fieldtype": "Datetime",
|
||||||
|
"label": "Passed Date"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"fieldname": "column_break_whlv",
|
||||||
|
"fieldtype": "Column Break"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"default": "0",
|
||||||
|
"fieldname": "passed",
|
||||||
|
"fieldtype": "Check",
|
||||||
|
"label": "Passed"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"grid_page_length": 50,
|
||||||
|
"index_web_pages_for_search": 1,
|
||||||
|
"links": [],
|
||||||
|
"modified": "2026-06-09 13:18:06.533455",
|
||||||
|
"modified_by": "support@seeraarabia.com",
|
||||||
|
"module": "SFDA Parser",
|
||||||
|
"name": "SFDA Entries",
|
||||||
|
"naming_rule": "By fieldname",
|
||||||
|
"owner": "Administrator",
|
||||||
|
"permissions": [
|
||||||
|
{
|
||||||
|
"create": 1,
|
||||||
|
"delete": 1,
|
||||||
|
"email": 1,
|
||||||
|
"export": 1,
|
||||||
|
"print": 1,
|
||||||
|
"read": 1,
|
||||||
|
"report": 1,
|
||||||
|
"role": "System Manager",
|
||||||
|
"share": 1,
|
||||||
|
"write": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"row_format": "Dynamic",
|
||||||
|
"rows_threshold_for_grid_search": 20,
|
||||||
|
"sort_field": "modified",
|
||||||
|
"sort_order": "DESC",
|
||||||
|
"states": []
|
||||||
|
}
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# Copyright (c) 2026, seyfert and contributors
|
||||||
|
# For license information, please see license.txt
|
||||||
|
|
||||||
|
from frappe.model.document import Document
|
||||||
|
|
||||||
|
|
||||||
|
class SFDAEntries(Document):
|
||||||
|
pass
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# Copyright (c) 2026, seyfert and Contributors
|
||||||
|
# See license.txt
|
||||||
|
|
||||||
|
from frappe.tests import IntegrationTestCase
|
||||||
|
|
||||||
|
|
||||||
|
class TestSFDAEntries(IntegrationTestCase):
|
||||||
|
pass
|
||||||
1
sfda_parser/sfda_scraper/__init__.py
Normal file
1
sfda_parser/sfda_scraper/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
# SFDA weekly recall scraper for sfda_parser
|
||||||
BIN
sfda_parser/sfda_scraper/__pycache__/__init__.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/__init__.cpython-310.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/detail_page.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/detail_page.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/detail_pdf.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/detail_pdf.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/http_client.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/http_client.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/importer.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/importer.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/jobs.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/jobs.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/reparse.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/reparse.cpython-310.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/weekly_list.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/weekly_list.cpython-310.pyc
Normal file
Binary file not shown.
BIN
sfda_parser/sfda_scraper/__pycache__/weekly_pdf.cpython-310.pyc
Normal file
BIN
sfda_parser/sfda_scraper/__pycache__/weekly_pdf.cpython-310.pyc
Normal file
Binary file not shown.
192
sfda_parser/sfda_scraper/detail_page.py
Normal file
192
sfda_parser/sfda_scraper/detail_page.py
Normal file
@ -0,0 +1,192 @@
|
|||||||
|
"""Fetch and parse ADE PublishDetails pages."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_pdf import parse_affected_device_list
|
||||||
|
from sfda_parser.sfda_scraper.http_client import ADE_BASE, create_session, request_with_retry
|
||||||
|
|
||||||
|
DOWNLOAD_PATTERN = re.compile(
|
||||||
|
r'formaction=["\'](?P<path>/BaseService/DownloadDocument\?download=[^"\']+)["\']',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
TOKEN_PATTERN = re.compile(
|
||||||
|
r'name=["\']__RequestVerificationToken["\'][^>]*value=["\']([^"\']+)["\']',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
PUBLISH_ID_PATTERN = re.compile(r"/Fsca/PublishDetails/(\d+)", re.IGNORECASE)
|
||||||
|
|
||||||
|
HTML_FIELD_MAP = {
|
||||||
|
"reference_number": "ReferenceNumber",
|
||||||
|
"manufacturer": "ManufacturerName",
|
||||||
|
"product_trade_name": "ProductTradeName",
|
||||||
|
"model": "ModelNbr",
|
||||||
|
"authorized_representative": "AuthorizedRepresentative",
|
||||||
|
"problem_reason": "ProblemReason",
|
||||||
|
"action": "Action",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class NoPdfDownloadLinkError(ValueError):
|
||||||
|
"""ADE PublishDetails page has no DownloadDocument link."""
|
||||||
|
|
||||||
|
|
||||||
|
def _field_value(soup: BeautifulSoup, field_name: str) -> str:
|
||||||
|
el = soup.find(attrs={"name": field_name})
|
||||||
|
if not el:
|
||||||
|
return ""
|
||||||
|
if el.name == "textarea":
|
||||||
|
return el.get_text(strip=True)
|
||||||
|
return (el.get("value") or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_download_path(html: str) -> str | None:
|
||||||
|
match = DOWNLOAD_PATTERN.search(html)
|
||||||
|
if match:
|
||||||
|
return match.group("path")
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
button = soup.select_one("button[formaction*='DownloadDocument']")
|
||||||
|
if button and button.get("formaction"):
|
||||||
|
return button["formaction"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_verification_token(html: str) -> str | None:
|
||||||
|
match = TOKEN_PATTERN.search(html)
|
||||||
|
if match:
|
||||||
|
return match.group(1)
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
inp = soup.find("input", {"name": "__RequestVerificationToken"})
|
||||||
|
if inp and inp.get("value"):
|
||||||
|
return inp["value"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_publish_details_html(html: str, detail_url: str = "") -> dict[str, Any]:
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
publish_id = ""
|
||||||
|
if detail_url:
|
||||||
|
match = PUBLISH_ID_PATTERN.search(detail_url)
|
||||||
|
if match:
|
||||||
|
publish_id = match.group(1)
|
||||||
|
|
||||||
|
download_path = _extract_download_path(html)
|
||||||
|
data: dict[str, Any] = {
|
||||||
|
"detail_url": detail_url,
|
||||||
|
"publish_id": publish_id,
|
||||||
|
"pdf_download_url": urljoin(ADE_BASE, download_path) if download_path else "",
|
||||||
|
}
|
||||||
|
for key, field_name in HTML_FIELD_MAP.items():
|
||||||
|
data[key] = _field_value(soup, field_name)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_publish_details_html(detail_url: str, session=None) -> dict[str, Any]:
|
||||||
|
session = session or create_session()
|
||||||
|
response = request_with_retry(session, "GET", detail_url, timeout=60)
|
||||||
|
return parse_publish_details_html(response.text, detail_url=detail_url)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_publish_details(
|
||||||
|
detail_url: str,
|
||||||
|
session=None,
|
||||||
|
include_device_list_from_pdf: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""ADE page fields; optionally parse Material/GTIN/Batch table from attached PDF."""
|
||||||
|
session = session or create_session()
|
||||||
|
result = fetch_publish_details_html(detail_url, session=session)
|
||||||
|
|
||||||
|
if include_device_list_from_pdf:
|
||||||
|
try:
|
||||||
|
pdf_bytes = download_detail_pdf(detail_url, session=session)
|
||||||
|
devices = parse_affected_device_list(pdf_bytes)
|
||||||
|
result["device_list_from_pdf"] = [d.as_dict() for d in devices]
|
||||||
|
result["device_list_from_pdf_count"] = len(devices)
|
||||||
|
except Exception as exc:
|
||||||
|
result["device_list_from_pdf"] = []
|
||||||
|
result["device_list_from_pdf_count"] = 0
|
||||||
|
result["pdf_parse_error"] = str(exc)
|
||||||
|
else:
|
||||||
|
result["device_list_from_pdf"] = []
|
||||||
|
result["device_list_from_pdf_count"] = 0
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_detail_url_for_ncmdr_ref(ncmdr_ref: str, session=None) -> str | None:
|
||||||
|
"""Resolve ADE URL from recent weekly PDFs when not stored on SFDA Entries."""
|
||||||
|
from sfda_parser.sfda_scraper.weekly_list import fetch_recent_weekly_alerts
|
||||||
|
from sfda_parser.sfda_scraper.weekly_pdf import parse_safety_alerts_from_pdf
|
||||||
|
|
||||||
|
session = session or create_session()
|
||||||
|
target = (ncmdr_ref or "").strip().upper()
|
||||||
|
for weekly in fetch_recent_weekly_alerts(session=session):
|
||||||
|
pdf_response = session.get(weekly.pdf_url, timeout=120)
|
||||||
|
pdf_response.raise_for_status()
|
||||||
|
for alert in parse_safety_alerts_from_pdf(pdf_response.content):
|
||||||
|
if alert.ncmdr_ref.strip().upper() == target:
|
||||||
|
return alert.detail_url
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _download_detail_pdf_from_html(
|
||||||
|
html: str, detail_url: str, session
|
||||||
|
) -> bytes:
|
||||||
|
download_path = _extract_download_path(html)
|
||||||
|
token = _extract_verification_token(html)
|
||||||
|
if not download_path:
|
||||||
|
raise NoPdfDownloadLinkError(f"No PDF download link on ADE page: {detail_url}")
|
||||||
|
|
||||||
|
download_url = urljoin(ADE_BASE, download_path)
|
||||||
|
headers = {"Referer": detail_url}
|
||||||
|
data = {}
|
||||||
|
if token:
|
||||||
|
data["__RequestVerificationToken"] = token
|
||||||
|
|
||||||
|
pdf_response = request_with_retry(
|
||||||
|
session,
|
||||||
|
"POST",
|
||||||
|
download_url,
|
||||||
|
data=data,
|
||||||
|
headers=headers,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
|
||||||
|
content_type = (pdf_response.headers.get("Content-Type") or "").lower()
|
||||||
|
if "pdf" not in content_type and not pdf_response.content.startswith(b"%PDF"):
|
||||||
|
raise ValueError(f"ADE download did not return a PDF for {detail_url}")
|
||||||
|
|
||||||
|
return pdf_response.content
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_ade_detail(
|
||||||
|
detail_url: str, session=None
|
||||||
|
) -> tuple[bytes | None, dict[str, Any]]:
|
||||||
|
"""Fetch ADE detail page; download PDF when available.
|
||||||
|
|
||||||
|
Returns (pdf_bytes or None, publish_details). PDF is None when the ADE page
|
||||||
|
has no DownloadDocument link.
|
||||||
|
"""
|
||||||
|
session = session or create_session()
|
||||||
|
response = request_with_retry(session, "GET", detail_url, timeout=60)
|
||||||
|
html = response.text
|
||||||
|
publish_details = parse_publish_details_html(html, detail_url=detail_url)
|
||||||
|
|
||||||
|
try:
|
||||||
|
pdf_bytes = _download_detail_pdf_from_html(html, detail_url, session)
|
||||||
|
except NoPdfDownloadLinkError:
|
||||||
|
return None, publish_details
|
||||||
|
|
||||||
|
return pdf_bytes, publish_details
|
||||||
|
|
||||||
|
|
||||||
|
def download_detail_pdf(detail_url: str, session=None) -> bytes:
|
||||||
|
pdf_bytes, _publish_details = fetch_ade_detail(detail_url, session=session)
|
||||||
|
if pdf_bytes is None:
|
||||||
|
raise NoPdfDownloadLinkError(f"No PDF download link on ADE page: {detail_url}")
|
||||||
|
return pdf_bytes
|
||||||
1295
sfda_parser/sfda_scraper/detail_pdf.py
Normal file
1295
sfda_parser/sfda_scraper/detail_pdf.py
Normal file
File diff suppressed because it is too large
Load Diff
74
sfda_parser/sfda_scraper/http_client.py
Normal file
74
sfda_parser/sfda_scraper/http_client.py
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
"""Shared HTTP session for SFDA / ADE scraping."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
DEFAULT_HEADERS = {
|
||||||
|
"User-Agent": (
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||||
|
),
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
SFDA_BASE = "https://www.sfda.gov.sa"
|
||||||
|
ADE_BASE = "https://ade.sfda.gov.sa"
|
||||||
|
|
||||||
|
# Rate-limit protection for ADE (DownloadDocument throttles after ~10 rapid calls).
|
||||||
|
ADE_REQUEST_DELAY_SECONDS = 3
|
||||||
|
ADE_MAX_RETRIES = 3
|
||||||
|
ADE_RETRY_BACKOFF_SECONDS = 30
|
||||||
|
ADE_RETRY_STATUS_CODES = {429, 503}
|
||||||
|
|
||||||
|
|
||||||
|
def create_session() -> requests.Session:
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers.update(DEFAULT_HEADERS)
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_wait_seconds(response: requests.Response, attempt: int) -> float:
|
||||||
|
retry_after = response.headers.get("Retry-After")
|
||||||
|
if retry_after:
|
||||||
|
try:
|
||||||
|
return max(float(retry_after), 1.0)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return ADE_RETRY_BACKOFF_SECONDS * (2**attempt)
|
||||||
|
|
||||||
|
|
||||||
|
def request_with_retry(
|
||||||
|
session: requests.Session,
|
||||||
|
method: str,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
max_retries: int = ADE_MAX_RETRIES,
|
||||||
|
retry_status_codes: set[int] | None = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> requests.Response:
|
||||||
|
"""Perform an HTTP request with exponential backoff on rate-limit responses."""
|
||||||
|
retry_status_codes = retry_status_codes or ADE_RETRY_STATUS_CODES
|
||||||
|
last_response: requests.Response | None = None
|
||||||
|
|
||||||
|
for attempt in range(max_retries + 1):
|
||||||
|
response = session.request(method, url, **kwargs)
|
||||||
|
last_response = response
|
||||||
|
|
||||||
|
if response.status_code not in retry_status_codes:
|
||||||
|
response.raise_for_status()
|
||||||
|
return response
|
||||||
|
|
||||||
|
if attempt >= max_retries:
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
wait_seconds = _retry_wait_seconds(response, attempt)
|
||||||
|
time.sleep(wait_seconds)
|
||||||
|
|
||||||
|
if last_response is not None:
|
||||||
|
last_response.raise_for_status()
|
||||||
|
raise requests.HTTPError(f"Request failed for {url}")
|
||||||
407
sfda_parser/sfda_scraper/importer.py
Normal file
407
sfda_parser/sfda_scraper/importer.py
Normal file
@ -0,0 +1,407 @@
|
|||||||
|
"""Import scraped SFDA alerts into SFDA Entries documents."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
from frappe.utils import now_datetime
|
||||||
|
from frappe.utils.file_manager import save_file
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_page import fetch_ade_detail
|
||||||
|
from sfda_parser.sfda_scraper.detail_pdf import parse_affected_device_list
|
||||||
|
from sfda_parser.sfda_scraper.http_client import ADE_REQUEST_DELAY_SECONDS, create_session
|
||||||
|
from sfda_parser.sfda_scraper.weekly_list import (
|
||||||
|
WEEKLY_BULLETINS_TO_IMPORT,
|
||||||
|
WeeklyAlertRow,
|
||||||
|
fetch_recent_weekly_alerts,
|
||||||
|
)
|
||||||
|
from sfda_parser.sfda_scraper.weekly_pdf import SafetyAlertRow, parse_safety_alerts_from_pdf
|
||||||
|
|
||||||
|
# Tracks which weekly PDF was last processed (persists across scheduled runs).
|
||||||
|
LAST_WEEKLY_PDF_KEY = "sfda_parser_last_processed_weekly_pdf_url"
|
||||||
|
FAILED_REFS_CACHE_KEY = "sfda_parser_failed_import_refs"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_ncmdr_ref(ncmdr_ref: str) -> str:
|
||||||
|
return (ncmdr_ref or "").strip().upper()
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_detail_pdf(doc_name: str, ncmdr_ref: str, pdf_bytes: bytes) -> str:
|
||||||
|
safe_ref = re.sub(r"[^\w\-]", "_", ncmdr_ref)
|
||||||
|
file_doc = save_file(
|
||||||
|
f"{safe_ref}-ade-detail.pdf",
|
||||||
|
pdf_bytes,
|
||||||
|
"SFDA Entries",
|
||||||
|
doc_name,
|
||||||
|
decode=False,
|
||||||
|
is_private=0,
|
||||||
|
)
|
||||||
|
return file_doc.file_url
|
||||||
|
|
||||||
|
|
||||||
|
def _set_detail_pdf_on_child_rows(doc_name: str, ncmdr_ref: str, file_url: str) -> None:
|
||||||
|
target = _normalize_ncmdr_ref(ncmdr_ref)
|
||||||
|
for row in frappe.get_all(
|
||||||
|
"SFDA Device Entries",
|
||||||
|
filters={"parent": doc_name, "parenttype": "SFDA Entries"},
|
||||||
|
fields=["name", "ncmdr_ref"],
|
||||||
|
):
|
||||||
|
if _normalize_ncmdr_ref(row.get("ncmdr_ref") or "") == target:
|
||||||
|
frappe.db.set_value("SFDA Device Entries", row.name, "detail_pdf", file_url)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_failed_refs() -> list[dict[str, Any]]:
|
||||||
|
return frappe.cache().get_value(FAILED_REFS_CACHE_KEY) or []
|
||||||
|
|
||||||
|
|
||||||
|
def _save_failed_refs(refs: list[dict[str, Any]]) -> None:
|
||||||
|
frappe.cache().set_value(FAILED_REFS_CACHE_KEY, refs)
|
||||||
|
|
||||||
|
|
||||||
|
def _add_failed_ref(
|
||||||
|
ncmdr_ref: str,
|
||||||
|
detail_url: str,
|
||||||
|
weekly_title: str,
|
||||||
|
weekly_date: date | None,
|
||||||
|
) -> None:
|
||||||
|
refs = _get_failed_refs()
|
||||||
|
existing = {_normalize_ncmdr_ref(item.get("ncmdr_ref") or "") for item in refs}
|
||||||
|
if _normalize_ncmdr_ref(ncmdr_ref) in existing:
|
||||||
|
return
|
||||||
|
refs.append(
|
||||||
|
{
|
||||||
|
"ncmdr_ref": ncmdr_ref,
|
||||||
|
"detail_url": detail_url,
|
||||||
|
"weekly_title": weekly_title,
|
||||||
|
"weekly_date": str(weekly_date) if weekly_date else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_save_failed_refs(refs)
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_failed_ref(ncmdr_ref: str) -> None:
|
||||||
|
target = _normalize_ncmdr_ref(ncmdr_ref)
|
||||||
|
refs = [
|
||||||
|
item
|
||||||
|
for item in _get_failed_refs()
|
||||||
|
if _normalize_ncmdr_ref(item.get("ncmdr_ref") or "") != target
|
||||||
|
]
|
||||||
|
_save_failed_refs(refs)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_weekly_entry_name(weekly_title: str, weekly_date: date | None) -> str | None:
|
||||||
|
filters: dict[str, Any] = {"title": weekly_title}
|
||||||
|
if weekly_date:
|
||||||
|
filters["date"] = weekly_date
|
||||||
|
return frappe.db.get_value("SFDA Entries", filters)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_existing_ncmdr_refs(doc_name: str) -> set[str]:
|
||||||
|
rows = frappe.get_all(
|
||||||
|
"SFDA Device Entries",
|
||||||
|
filters={"parent": doc_name, "parenttype": "SFDA Entries"},
|
||||||
|
pluck="ncmdr_ref",
|
||||||
|
)
|
||||||
|
return {_normalize_ncmdr_ref(ref) for ref in rows if ref}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_device_rows_for_alert(
|
||||||
|
alert: SafetyAlertRow,
|
||||||
|
*,
|
||||||
|
manufacturer: str,
|
||||||
|
devices: list,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
ncmdr_ref = (alert.ncmdr_ref or "").strip()
|
||||||
|
base = {
|
||||||
|
"doctype": "SFDA Device Entries",
|
||||||
|
"ncmdr_ref": ncmdr_ref,
|
||||||
|
"manufacturer": manufacturer,
|
||||||
|
"ade_detail_url": alert.detail_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
if devices:
|
||||||
|
return [{**base, **device.as_dict()} for device in devices]
|
||||||
|
|
||||||
|
return [base]
|
||||||
|
|
||||||
|
|
||||||
|
def _import_single_alert_into_weekly_entry(
|
||||||
|
alert: SafetyAlertRow,
|
||||||
|
*,
|
||||||
|
weekly_title: str,
|
||||||
|
weekly_date: date | None,
|
||||||
|
session,
|
||||||
|
summary: dict,
|
||||||
|
doc,
|
||||||
|
existing_ncmdr_refs: set[str],
|
||||||
|
pending_pdfs: list[tuple[str, bytes]],
|
||||||
|
) -> bool:
|
||||||
|
"""Fetch one alert and append child rows to the weekly SFDA Entries doc."""
|
||||||
|
ncmdr_ref = (alert.ncmdr_ref or "").strip()
|
||||||
|
if not ncmdr_ref:
|
||||||
|
return False
|
||||||
|
|
||||||
|
normalized_ref = _normalize_ncmdr_ref(ncmdr_ref)
|
||||||
|
if normalized_ref in existing_ncmdr_refs:
|
||||||
|
summary["skipped_existing"] += 1
|
||||||
|
_remove_failed_ref(ncmdr_ref)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
detail_pdf, publish_details = fetch_ade_detail(alert.detail_url, session=session)
|
||||||
|
manufacturer = (publish_details.get("manufacturer") or "").strip()
|
||||||
|
|
||||||
|
devices = parse_affected_device_list(detail_pdf) if detail_pdf else []
|
||||||
|
if not detail_pdf:
|
||||||
|
warning_msg = (
|
||||||
|
f"{ncmdr_ref} ({alert.detail_url}): No PDF on ADE page; "
|
||||||
|
"created child row from HTML fields only"
|
||||||
|
)
|
||||||
|
summary["warnings"].append(warning_msg)
|
||||||
|
frappe.logger("sfda_scraper").warning(warning_msg)
|
||||||
|
|
||||||
|
for row in _build_device_rows_for_alert(
|
||||||
|
alert,
|
||||||
|
manufacturer=manufacturer,
|
||||||
|
devices=devices,
|
||||||
|
):
|
||||||
|
doc.append("device_list", row)
|
||||||
|
|
||||||
|
if detail_pdf:
|
||||||
|
pending_pdfs.append((ncmdr_ref, detail_pdf))
|
||||||
|
summary["pdfs_saved"] += 1
|
||||||
|
|
||||||
|
existing_ncmdr_refs.add(normalized_ref)
|
||||||
|
summary["alerts_imported"] += 1
|
||||||
|
_remove_failed_ref(ncmdr_ref)
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
error_msg = f"{ncmdr_ref} ({alert.detail_url}): {exc}"
|
||||||
|
summary["errors"].append(error_msg)
|
||||||
|
_add_failed_ref(ncmdr_ref, alert.detail_url, weekly_title, weekly_date)
|
||||||
|
frappe.log_error(
|
||||||
|
title="SFDA Scraper: Alert import failed",
|
||||||
|
message=error_msg,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _get_or_create_weekly_doc(weekly_title: str, weekly_date: date | None):
|
||||||
|
existing_name = _find_weekly_entry_name(weekly_title, weekly_date)
|
||||||
|
if existing_name:
|
||||||
|
return frappe.get_doc("SFDA Entries", existing_name), _get_existing_ncmdr_refs(
|
||||||
|
existing_name
|
||||||
|
), True
|
||||||
|
|
||||||
|
doc = frappe.get_doc(
|
||||||
|
{
|
||||||
|
"doctype": "SFDA Entries",
|
||||||
|
"title": weekly_title,
|
||||||
|
"date": weekly_date,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return doc, set(), False
|
||||||
|
|
||||||
|
|
||||||
|
def _save_weekly_doc(doc, is_existing: bool) -> None:
|
||||||
|
if is_existing:
|
||||||
|
doc.save(ignore_permissions=True)
|
||||||
|
else:
|
||||||
|
doc.insert(ignore_permissions=True)
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_pending_pdfs(doc_name: str, pending_pdfs: list[tuple[str, bytes]]) -> None:
|
||||||
|
for ncmdr_ref, pdf_bytes in pending_pdfs:
|
||||||
|
file_url = _attach_detail_pdf(doc_name, ncmdr_ref, pdf_bytes)
|
||||||
|
_set_detail_pdf_on_child_rows(doc_name, ncmdr_ref, file_url)
|
||||||
|
if pending_pdfs:
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _import_weekly_bulletin(
|
||||||
|
weekly: WeeklyAlertRow,
|
||||||
|
alerts: list[SafetyAlertRow],
|
||||||
|
*,
|
||||||
|
session,
|
||||||
|
summary: dict,
|
||||||
|
) -> None:
|
||||||
|
doc, existing_ncmdr_refs, is_existing = _get_or_create_weekly_doc(
|
||||||
|
weekly.title,
|
||||||
|
weekly.alert_date,
|
||||||
|
)
|
||||||
|
pending_pdfs: list[tuple[str, bytes]] = []
|
||||||
|
imported_any = False
|
||||||
|
|
||||||
|
for alert in alerts:
|
||||||
|
if _import_single_alert_into_weekly_entry(
|
||||||
|
alert,
|
||||||
|
weekly_title=weekly.title,
|
||||||
|
weekly_date=weekly.alert_date,
|
||||||
|
session=session,
|
||||||
|
summary=summary,
|
||||||
|
doc=doc,
|
||||||
|
existing_ncmdr_refs=existing_ncmdr_refs,
|
||||||
|
pending_pdfs=pending_pdfs,
|
||||||
|
):
|
||||||
|
imported_any = True
|
||||||
|
time.sleep(ADE_REQUEST_DELAY_SECONDS)
|
||||||
|
|
||||||
|
if not imported_any:
|
||||||
|
return
|
||||||
|
|
||||||
|
_save_weekly_doc(doc, is_existing)
|
||||||
|
_attach_pending_pdfs(doc.name, pending_pdfs)
|
||||||
|
|
||||||
|
if not is_existing:
|
||||||
|
summary["created"] += 1
|
||||||
|
else:
|
||||||
|
summary["updated"] += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_failed_refs(session, summary: dict) -> None:
|
||||||
|
"""Retry alerts that failed on a previous run (e.g. ADE 429 rate limits)."""
|
||||||
|
failed_refs = _get_failed_refs()
|
||||||
|
if not failed_refs:
|
||||||
|
return
|
||||||
|
|
||||||
|
summary["failed_refs_queued"] = len(failed_refs)
|
||||||
|
frappe.logger("sfda_scraper").info(
|
||||||
|
"SFDA scraper retrying %s previously failed alert(s)",
|
||||||
|
len(failed_refs),
|
||||||
|
)
|
||||||
|
|
||||||
|
weekly_groups: dict[tuple[str, str | None], list[dict[str, Any]]] = {}
|
||||||
|
for item in failed_refs:
|
||||||
|
weekly_date = item.get("weekly_date")
|
||||||
|
weekly_groups.setdefault(
|
||||||
|
(item.get("weekly_title") or "", weekly_date),
|
||||||
|
[],
|
||||||
|
).append(item)
|
||||||
|
|
||||||
|
for (weekly_title, weekly_date_str), items in weekly_groups.items():
|
||||||
|
weekly_date = None
|
||||||
|
if weekly_date_str:
|
||||||
|
try:
|
||||||
|
weekly_date = date.fromisoformat(weekly_date_str)
|
||||||
|
except ValueError:
|
||||||
|
weekly_date = None
|
||||||
|
|
||||||
|
weekly = WeeklyAlertRow(
|
||||||
|
title=weekly_title,
|
||||||
|
alert_date=weekly_date,
|
||||||
|
pdf_url="",
|
||||||
|
)
|
||||||
|
alerts = [
|
||||||
|
SafetyAlertRow(
|
||||||
|
ncmdr_ref=item.get("ncmdr_ref") or "",
|
||||||
|
detail_url=item.get("detail_url") or "",
|
||||||
|
)
|
||||||
|
for item in items
|
||||||
|
]
|
||||||
|
_import_weekly_bulletin(weekly, alerts, session=session, summary=summary)
|
||||||
|
|
||||||
|
|
||||||
|
def import_latest_weekly_alerts() -> dict:
|
||||||
|
"""
|
||||||
|
Recurring job: reads the most recent weekly bulletin(s) from SFDA and imports
|
||||||
|
alerts into one SFDA Entries document per weekly bulletin.
|
||||||
|
|
||||||
|
Each scheduled run:
|
||||||
|
- Retries previously failed refs from the cache queue
|
||||||
|
- Fetches https://www.sfda.gov.sa/en/weekly-alert
|
||||||
|
- Uses the last WEEKLY_BULLETINS_TO_IMPORT weekly PDFs
|
||||||
|
- Creates or updates one SFDA Entries doc per week with child rows per alert
|
||||||
|
- Skips NCMDR refs already present in that week's device_list
|
||||||
|
"""
|
||||||
|
session = create_session()
|
||||||
|
summary = {
|
||||||
|
"weekly_bulletins_to_import": WEEKLY_BULLETINS_TO_IMPORT,
|
||||||
|
"weekly_title": "",
|
||||||
|
"weekly_date": None,
|
||||||
|
"weekly_pdf_url": "",
|
||||||
|
"weekly_titles": [],
|
||||||
|
"alerts_found": 0,
|
||||||
|
"alerts_imported": 0,
|
||||||
|
"created": 0,
|
||||||
|
"updated": 0,
|
||||||
|
"pdfs_saved": 0,
|
||||||
|
"skipped_existing": 0,
|
||||||
|
"weekly_already_processed": False,
|
||||||
|
"failed_refs_queued": 0,
|
||||||
|
"warnings": [],
|
||||||
|
"errors": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
_retry_failed_refs(session, summary)
|
||||||
|
|
||||||
|
weeklies = fetch_recent_weekly_alerts(session=session)
|
||||||
|
if not weeklies:
|
||||||
|
frappe.log_error(
|
||||||
|
title="SFDA Scraper: No weekly bulletins found",
|
||||||
|
message="https://www.sfda.gov.sa/en/weekly-alert",
|
||||||
|
)
|
||||||
|
_log_run_summary(summary)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
weekly_newest = weeklies[0]
|
||||||
|
summary["weekly_title"] = weekly_newest.title
|
||||||
|
summary["weekly_date"] = (
|
||||||
|
str(weekly_newest.alert_date) if weekly_newest.alert_date else None
|
||||||
|
)
|
||||||
|
summary["weekly_pdf_url"] = weekly_newest.pdf_url
|
||||||
|
summary["weekly_titles"] = [w.title for w in weeklies]
|
||||||
|
|
||||||
|
last_pdf_url = frappe.db.get_default(LAST_WEEKLY_PDF_KEY)
|
||||||
|
|
||||||
|
# Oldest week first so backfill order matches bulletin dates.
|
||||||
|
for weekly in reversed(weeklies):
|
||||||
|
pdf_response = session.get(weekly.pdf_url, timeout=120)
|
||||||
|
pdf_response.raise_for_status()
|
||||||
|
alerts = parse_safety_alerts_from_pdf(pdf_response.content)
|
||||||
|
|
||||||
|
if not alerts:
|
||||||
|
frappe.log_error(
|
||||||
|
title="SFDA Scraper: No alerts in weekly PDF",
|
||||||
|
message=f"URL: {weekly.pdf_url}",
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
summary["alerts_found"] += len(alerts)
|
||||||
|
_import_weekly_bulletin(weekly, alerts, session=session, summary=summary)
|
||||||
|
|
||||||
|
# Same newest weekly PDF as last run and every alert already in ERP
|
||||||
|
if (
|
||||||
|
last_pdf_url == weekly_newest.pdf_url
|
||||||
|
and summary["created"] == 0
|
||||||
|
and summary["updated"] == 0
|
||||||
|
and summary["skipped_existing"] == summary["alerts_found"]
|
||||||
|
and summary["alerts_found"] > 0
|
||||||
|
):
|
||||||
|
summary["weekly_already_processed"] = True
|
||||||
|
|
||||||
|
# Track newest bulletin URL (detect when SFDA publishes a new top-row PDF)
|
||||||
|
frappe.db.set_default(LAST_WEEKLY_PDF_KEY, weekly_newest.pdf_url)
|
||||||
|
|
||||||
|
_log_run_summary(summary)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def _log_run_summary(summary: dict) -> None:
|
||||||
|
frappe.logger("sfda_scraper").info(
|
||||||
|
"SFDA scheduled import at %s — latest weekly bulletin '%s' (%s): %s",
|
||||||
|
now_datetime(),
|
||||||
|
summary.get("weekly_title"),
|
||||||
|
summary.get("weekly_date"),
|
||||||
|
summary,
|
||||||
|
)
|
||||||
|
if summary.get("errors"):
|
||||||
|
frappe.log_error(
|
||||||
|
title="SFDA Scraper: Run completed with errors",
|
||||||
|
message=frappe.as_json(summary, indent=2),
|
||||||
|
)
|
||||||
25
sfda_parser/sfda_scraper/jobs.py
Normal file
25
sfda_parser/sfda_scraper/jobs.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
"""Scheduled and manual entry points for the SFDA scraper."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.importer import import_latest_weekly_alerts
|
||||||
|
|
||||||
|
|
||||||
|
def run_weekly_sfda_import_scheduled():
|
||||||
|
"""
|
||||||
|
Runs automatically every Monday (see hooks.py cron).
|
||||||
|
|
||||||
|
Each execution:
|
||||||
|
1. Opens SFDA weekly-alert page
|
||||||
|
2. Picks the **latest** bulletin (top / newest date)
|
||||||
|
3. Imports new recalls into SFDA Entries (skips existing NCMDR refs)
|
||||||
|
|
||||||
|
When SFDA publishes a new weekly PDF, the table top row changes;
|
||||||
|
the next run imports only alerts that are not yet in ERP.
|
||||||
|
"""
|
||||||
|
frappe.logger("sfda_scraper").info("SFDA weekly scheduled job started")
|
||||||
|
summary = import_latest_weekly_alerts()
|
||||||
|
frappe.logger("sfda_scraper").info("SFDA weekly scheduled job finished: %s", summary)
|
||||||
|
return summary
|
||||||
140
sfda_parser/sfda_scraper/reparse.py
Normal file
140
sfda_parser/sfda_scraper/reparse.py
Normal file
@ -0,0 +1,140 @@
|
|||||||
|
"""Re-parse device_list child rows from stored ADE detail PDF attachments."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import frappe
|
||||||
|
from frappe.utils.file_manager import get_file_path
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_pdf import parse_affected_device_list
|
||||||
|
|
||||||
|
|
||||||
|
def _read_detail_pdf_bytes(file_url: str) -> bytes:
|
||||||
|
file_name = file_url.split("/files/")[-1]
|
||||||
|
path = get_file_path(file_name)
|
||||||
|
if not path or not os.path.isfile(path):
|
||||||
|
site_path = frappe.get_site_path("public", "files", file_name)
|
||||||
|
if os.path.isfile(site_path):
|
||||||
|
path = site_path
|
||||||
|
else:
|
||||||
|
raise FileNotFoundError(f"PDF not found for {file_url}")
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_ncmdr_ref(ncmdr_ref: str) -> str:
|
||||||
|
return (ncmdr_ref or "").strip().upper()
|
||||||
|
|
||||||
|
|
||||||
|
def _group_child_rows_by_ncmdr_ref(doc) -> dict[str, dict[str, str]]:
|
||||||
|
groups: dict[str, dict[str, str]] = {}
|
||||||
|
for row in doc.device_list or []:
|
||||||
|
ref = (row.ncmdr_ref or "").strip()
|
||||||
|
if not ref:
|
||||||
|
continue
|
||||||
|
key = _normalize_ncmdr_ref(ref)
|
||||||
|
if key not in groups:
|
||||||
|
groups[key] = {
|
||||||
|
"ncmdr_ref": ref,
|
||||||
|
"manufacturer": row.manufacturer or "",
|
||||||
|
"detail_pdf": row.detail_pdf or "",
|
||||||
|
"ade_detail_url": row.ade_detail_url or "",
|
||||||
|
}
|
||||||
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
def reparse_sfda_entry_device_list(doc_name: str) -> dict:
|
||||||
|
"""Rebuild device_list child rows from stored detail_pdf on each NCMDR group."""
|
||||||
|
doc = frappe.get_doc("SFDA Entries", doc_name)
|
||||||
|
groups = _group_child_rows_by_ncmdr_ref(doc)
|
||||||
|
if not groups:
|
||||||
|
return {"name": doc_name, "device_rows": 0, "skipped": "no_child_rows"}
|
||||||
|
|
||||||
|
rebuilt_rows: list[dict] = []
|
||||||
|
total_device_rows = 0
|
||||||
|
|
||||||
|
for group in groups.values():
|
||||||
|
detail_pdf = group.get("detail_pdf") or ""
|
||||||
|
if not detail_pdf:
|
||||||
|
rebuilt_rows.append(
|
||||||
|
{
|
||||||
|
"doctype": "SFDA Device Entries",
|
||||||
|
"ncmdr_ref": group["ncmdr_ref"],
|
||||||
|
"manufacturer": group["manufacturer"],
|
||||||
|
"ade_detail_url": group["ade_detail_url"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
pdf_bytes = _read_detail_pdf_bytes(detail_pdf)
|
||||||
|
devices = parse_affected_device_list(pdf_bytes)
|
||||||
|
if devices:
|
||||||
|
for device in devices:
|
||||||
|
rebuilt_rows.append(
|
||||||
|
{
|
||||||
|
"doctype": "SFDA Device Entries",
|
||||||
|
"ncmdr_ref": group["ncmdr_ref"],
|
||||||
|
"manufacturer": group["manufacturer"],
|
||||||
|
"detail_pdf": detail_pdf,
|
||||||
|
"ade_detail_url": group["ade_detail_url"],
|
||||||
|
**device.as_dict(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
total_device_rows += len(devices)
|
||||||
|
else:
|
||||||
|
rebuilt_rows.append(
|
||||||
|
{
|
||||||
|
"doctype": "SFDA Device Entries",
|
||||||
|
"ncmdr_ref": group["ncmdr_ref"],
|
||||||
|
"manufacturer": group["manufacturer"],
|
||||||
|
"detail_pdf": detail_pdf,
|
||||||
|
"ade_detail_url": group["ade_detail_url"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
doc.set("device_list", [])
|
||||||
|
for row in rebuilt_rows:
|
||||||
|
doc.append("device_list", row)
|
||||||
|
|
||||||
|
doc.save(ignore_permissions=True)
|
||||||
|
frappe.db.commit()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"name": doc_name,
|
||||||
|
"ncmdr_groups": len(groups),
|
||||||
|
"device_rows": total_device_rows or len(rebuilt_rows),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def reparse_all_stored_pdf_device_lists(sfda_entries_name: str | None = None) -> dict:
|
||||||
|
"""Re-parse device_list from child-row detail_pdf for one or all SFDA Entries."""
|
||||||
|
if sfda_entries_name:
|
||||||
|
names = [sfda_entries_name]
|
||||||
|
else:
|
||||||
|
names = frappe.db.sql_list(
|
||||||
|
"""
|
||||||
|
select distinct parent
|
||||||
|
from `tabSFDA Device Entries`
|
||||||
|
where ifnull(detail_pdf, '') != ''
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"processed": 0,
|
||||||
|
"updated": 0,
|
||||||
|
"total_device_rows": 0,
|
||||||
|
"errors": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for name in names:
|
||||||
|
summary["processed"] += 1
|
||||||
|
try:
|
||||||
|
result = reparse_sfda_entry_device_list(name)
|
||||||
|
summary["updated"] += 1
|
||||||
|
summary["total_device_rows"] += result.get("device_rows", 0)
|
||||||
|
except Exception as exc:
|
||||||
|
frappe.db.rollback()
|
||||||
|
summary["errors"].append(f"{name}: {exc}")
|
||||||
|
|
||||||
|
return summary
|
||||||
158
sfda_parser/sfda_scraper/test_detail_pdf_expand.py
Normal file
158
sfda_parser/sfda_scraper/test_detail_pdf_expand.py
Normal file
@ -0,0 +1,158 @@
|
|||||||
|
"""Tests for splitting comma-separated device identifiers into separate rows."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_pdf import (
|
||||||
|
DeviceRow,
|
||||||
|
_expand_device_row,
|
||||||
|
_expand_device_rows,
|
||||||
|
_split_multi_values,
|
||||||
|
parse_affected_device_list,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSplitMultiValues(unittest.TestCase):
|
||||||
|
def test_splits_comma_separated_values(self):
|
||||||
|
self.assertEqual(
|
||||||
|
_split_multi_values("24B0729, 25B0770, 25H0171, 26B0542"),
|
||||||
|
["24B0729", "25B0770", "25H0171", "26B0542"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_single_value_unchanged(self):
|
||||||
|
self.assertEqual(_split_multi_values("24B0729"), ["24B0729"])
|
||||||
|
|
||||||
|
def test_splits_semicolon_separated_values(self):
|
||||||
|
self.assertEqual(_split_multi_values("A1; A2; A3"), ["A1", "A2", "A3"])
|
||||||
|
|
||||||
|
def test_splits_space_separated_numeric_ids(self):
|
||||||
|
value = (
|
||||||
|
"0231547054 0231591827 0231758880 0231758898 0231758932 "
|
||||||
|
"0231792414 0231792421 0231912903 0232670901 0232709219 "
|
||||||
|
"0232709254 0232744928 0232784925 0232784989 0232805458 0232837110"
|
||||||
|
)
|
||||||
|
parts = _split_multi_values(value)
|
||||||
|
self.assertEqual(len(parts), 16)
|
||||||
|
self.assertEqual(parts[0], "0231547054")
|
||||||
|
self.assertEqual(parts[-1], "0232837110")
|
||||||
|
|
||||||
|
def test_does_not_split_prose_with_spaces(self):
|
||||||
|
self.assertEqual(
|
||||||
|
_split_multi_values("Surgical Instrument Kit"),
|
||||||
|
["Surgical Instrument Kit"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestExpandDeviceRow(unittest.TestCase):
|
||||||
|
def _row(self, **kwargs) -> DeviceRow:
|
||||||
|
base = {
|
||||||
|
"material": "Test Device",
|
||||||
|
"material_description": "Description",
|
||||||
|
"catalog_number": "",
|
||||||
|
"udi": "",
|
||||||
|
"serial_no": "",
|
||||||
|
"gstn": "",
|
||||||
|
"batch": "",
|
||||||
|
}
|
||||||
|
base.update(kwargs)
|
||||||
|
return DeviceRow(**base)
|
||||||
|
|
||||||
|
def test_expands_serial_numbers_into_separate_rows(self):
|
||||||
|
rows = _expand_device_row(
|
||||||
|
self._row(serial_no="24B0729, 25B0770, 25H0171, 26B0542")
|
||||||
|
)
|
||||||
|
self.assertEqual(len(rows), 4)
|
||||||
|
self.assertEqual([row.serial_no for row in rows], [
|
||||||
|
"24B0729",
|
||||||
|
"25B0770",
|
||||||
|
"25H0171",
|
||||||
|
"26B0542",
|
||||||
|
])
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row.material, "Test Device")
|
||||||
|
self.assertEqual(row.material_description, "Description")
|
||||||
|
|
||||||
|
def test_expands_catalog_numbers_into_separate_rows(self):
|
||||||
|
rows = _expand_device_row(self._row(catalog_number="CAT-1, CAT-2, CAT-3"))
|
||||||
|
self.assertEqual(len(rows), 3)
|
||||||
|
self.assertEqual([row.catalog_number for row in rows], ["CAT-1", "CAT-2", "CAT-3"])
|
||||||
|
|
||||||
|
def test_expands_batch_values_into_separate_rows(self):
|
||||||
|
rows = _expand_device_row(self._row(batch="LOT1, LOT2"))
|
||||||
|
self.assertEqual(len(rows), 2)
|
||||||
|
self.assertEqual([row.batch for row in rows], ["LOT1", "LOT2"])
|
||||||
|
|
||||||
|
def test_zips_serial_and_catalog_when_counts_match(self):
|
||||||
|
rows = _expand_device_row(
|
||||||
|
self._row(serial_no="S1, S2", catalog_number="C1, C2")
|
||||||
|
)
|
||||||
|
self.assertEqual(len(rows), 2)
|
||||||
|
self.assertEqual(rows[0].serial_no, "S1")
|
||||||
|
self.assertEqual(rows[0].catalog_number, "C1")
|
||||||
|
self.assertEqual(rows[1].serial_no, "S2")
|
||||||
|
self.assertEqual(rows[1].catalog_number, "C2")
|
||||||
|
|
||||||
|
def test_single_value_row_unchanged(self):
|
||||||
|
row = self._row(serial_no="ONLY-ONE", catalog_number="CAT-9")
|
||||||
|
self.assertEqual(_expand_device_row(row), [row])
|
||||||
|
|
||||||
|
def test_expand_device_rows_flattens_list(self):
|
||||||
|
devices = [
|
||||||
|
self._row(serial_no="A, B"),
|
||||||
|
self._row(catalog_number="X, Y, Z"),
|
||||||
|
]
|
||||||
|
expanded = _expand_device_rows(devices)
|
||||||
|
self.assertEqual(len(expanded), 5)
|
||||||
|
|
||||||
|
def test_expands_space_separated_udi_values(self):
|
||||||
|
udi_value = "0231547054 0231591827 0231758880"
|
||||||
|
rows = _expand_device_row(self._row(udi=udi_value, material="Monitor"))
|
||||||
|
self.assertEqual(len(rows), 3)
|
||||||
|
self.assertEqual(rows[1].udi, "0231591827")
|
||||||
|
self.assertEqual(rows[2].material, "Monitor")
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseAffectedDeviceListExpansion(unittest.TestCase):
|
||||||
|
def test_parse_expands_comma_separated_serials_from_table_like_input(self):
|
||||||
|
# Minimal synthetic check via direct expansion path used by parser output.
|
||||||
|
devices = _expand_device_rows([
|
||||||
|
DeviceRow(
|
||||||
|
material="Pump",
|
||||||
|
material_description="",
|
||||||
|
catalog_number="REF-100",
|
||||||
|
udi="",
|
||||||
|
serial_no="24B0729, 25B0770, 25H0171, 26B0542",
|
||||||
|
gstn="",
|
||||||
|
batch="",
|
||||||
|
)
|
||||||
|
])
|
||||||
|
self.assertEqual(len(devices), 4)
|
||||||
|
self.assertEqual(devices[0].catalog_number, "REF-100")
|
||||||
|
self.assertEqual(devices[-1].serial_no, "26B0542")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSparseTablePdfExpansion(unittest.TestCase):
|
||||||
|
def test_medtronic_pdf_expands_sparse_table_lot_lists(self):
|
||||||
|
pdf_path = os.path.abspath(
|
||||||
|
os.path.join(
|
||||||
|
os.path.dirname(__file__),
|
||||||
|
"../../../../sites/erp.seeraunified.com/public/files/"
|
||||||
|
"SA-28-05-26-1408-ade-detailad786ead786e.pdf",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not os.path.isfile(pdf_path):
|
||||||
|
self.skipTest(f"fixture PDF not found: {pdf_path}")
|
||||||
|
|
||||||
|
with open(pdf_path, "rb") as pdf_file:
|
||||||
|
devices = parse_affected_device_list(pdf_file.read())
|
||||||
|
|
||||||
|
self.assertGreaterEqual(len(devices), 40)
|
||||||
|
serial_numbers = [device.serial_no for device in devices if device.serial_no]
|
||||||
|
self.assertIn("0012508285", serial_numbers)
|
||||||
|
self.assertIn("0013272796", serial_numbers)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
176
sfda_parser/sfda_scraper/test_detail_pdf_sanitize.py
Normal file
176
sfda_parser/sfda_scraper/test_detail_pdf_sanitize.py
Normal file
@ -0,0 +1,176 @@
|
|||||||
|
"""Tests for material/serial sanitization in ADE detail PDF parsing."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.detail_pdf import (
|
||||||
|
DeviceRow,
|
||||||
|
_clean_material_value,
|
||||||
|
_clean_serial_value,
|
||||||
|
_is_likely_date_value,
|
||||||
|
_is_likely_prose,
|
||||||
|
_paired_rows_from_summary,
|
||||||
|
_parse_summary_kv,
|
||||||
|
_sanitize_device_row,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMaterialCleaning(unittest.TestCase):
|
||||||
|
def test_strips_product_concerned_prefix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
_clean_material_value("of the product concerned: TU2000 TRYTABLE"),
|
||||||
|
"TU2000 TRYTABLE",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_strips_trade_name_product_concerned_prefix(self):
|
||||||
|
self.assertEqual(
|
||||||
|
_clean_material_value("Trade name of the product concerned: TU2000 TRYTABLE"),
|
||||||
|
"TU2000 TRYTABLE",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_parse_summary_trade_name_line(self):
|
||||||
|
lines = ["Trade name of the product concerned: TU2000 TRYTABLE"]
|
||||||
|
summary = _parse_summary_kv(lines)
|
||||||
|
self.assertEqual(summary.get("material"), "TU2000 TRYTABLE")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerialDateRejection(unittest.TestCase):
|
||||||
|
def test_detects_day_month_year_dates(self):
|
||||||
|
self.assertTrue(_is_likely_date_value("21-APR-2025"))
|
||||||
|
self.assertTrue(_is_likely_date_value("21/APR/2025"))
|
||||||
|
|
||||||
|
def test_allows_real_serial_numbers(self):
|
||||||
|
self.assertFalse(_is_likely_date_value("24B0729"))
|
||||||
|
self.assertFalse(_is_likely_date_value("TSMICS1"))
|
||||||
|
|
||||||
|
def test_clean_serial_rejects_dates(self):
|
||||||
|
self.assertEqual(_clean_serial_value("21-APR-2025"), "")
|
||||||
|
self.assertEqual(_clean_serial_value("ABC12345"), "ABC12345")
|
||||||
|
|
||||||
|
def test_sanitize_device_row_clears_date_serial(self):
|
||||||
|
row = DeviceRow(
|
||||||
|
material="Pump",
|
||||||
|
material_description="",
|
||||||
|
catalog_number="",
|
||||||
|
udi="",
|
||||||
|
serial_no="21-APR-2025",
|
||||||
|
gstn="",
|
||||||
|
batch="",
|
||||||
|
)
|
||||||
|
cleaned = _sanitize_device_row(row)
|
||||||
|
self.assertEqual(cleaned.serial_no, "")
|
||||||
|
|
||||||
|
|
||||||
|
class TestPairedUdiSn(unittest.TestCase):
|
||||||
|
def test_pairs_udi_and_sn_by_position(self):
|
||||||
|
udi_values = (
|
||||||
|
"04020012, 04020014, 04020017, 04020018, 04020026, 04020021, "
|
||||||
|
"04020009, 04020010, 04020011, 04020001, 04020025, 04020013, "
|
||||||
|
"04020003, 04020005, 04020016, 04020023, 04020028, 04020008, "
|
||||||
|
"04020020, 04020019, 04020015, 04020024, 04020022, 04020006, "
|
||||||
|
"04020027, 04020029, 04020030, 04020031"
|
||||||
|
)
|
||||||
|
sn_values = (
|
||||||
|
"32090012, 32090014, 32090017, 32090018, 32090026, 32090021, "
|
||||||
|
"32090009, 32090010, 32090011, TR05617, 32090025, 32090013, "
|
||||||
|
"32090003, 302090005, 32090016, 32090023, 32090028, 32090008, "
|
||||||
|
"32090020, 32090019, 32090015, 32090024, 32090022, 302090006, "
|
||||||
|
"32090027, 32090029, 32090030, 32090031"
|
||||||
|
)
|
||||||
|
summary = {
|
||||||
|
"material": "TU2000 TRYTABLE",
|
||||||
|
"udi": udi_values,
|
||||||
|
"serial_no": sn_values,
|
||||||
|
}
|
||||||
|
rows = _paired_rows_from_summary(summary)
|
||||||
|
self.assertEqual(len(rows), 28)
|
||||||
|
self.assertEqual(rows[0].udi, "04020012")
|
||||||
|
self.assertEqual(rows[0].serial_no, "32090012")
|
||||||
|
self.assertEqual(rows[1].udi, "04020014")
|
||||||
|
self.assertEqual(rows[1].serial_no, "32090014")
|
||||||
|
self.assertEqual(rows[9].udi, "04020001")
|
||||||
|
self.assertEqual(rows[9].serial_no, "TR05617")
|
||||||
|
self.assertEqual(rows[27].udi, "04020031")
|
||||||
|
self.assertEqual(rows[27].serial_no, "32090031")
|
||||||
|
self.assertEqual(rows[0].material, "TU2000 TRYTABLE")
|
||||||
|
|
||||||
|
def test_parse_summary_kv_stops_before_narrative(self):
|
||||||
|
lines = [
|
||||||
|
"Trade name of the product concerned: TU2000 TRYTABLE",
|
||||||
|
"UDI-DI: 04020012, 04020014",
|
||||||
|
"SN: 32090012, 32090014",
|
||||||
|
"Dear Customer",
|
||||||
|
"The manufacturer has identified an error in the user manual.",
|
||||||
|
]
|
||||||
|
summary = _parse_summary_kv(lines)
|
||||||
|
self.assertEqual(summary.get("udi"), "04020012, 04020014")
|
||||||
|
self.assertEqual(summary.get("serial_no"), "32090012, 32090014")
|
||||||
|
self.assertNotIn("Dear Customer", summary.get("serial_no", ""))
|
||||||
|
|
||||||
|
def test_parse_summary_kv_multiline_udi_sn_lists(self):
|
||||||
|
lines = [
|
||||||
|
"Trade name of the product concerned: TU2000 TRYTABLE",
|
||||||
|
"UDI-DI: 04020012, 04020014, 04020017, 04020018, 04020026, 04020021, 04020009,",
|
||||||
|
"04020010, 04020011, 04020001, 04020025, 04020013, 04020003, 04020005, 04020016,",
|
||||||
|
"04020023, 04020028, 04020008, 04020020, 04020019, 04020015, 04020024, 04020022,",
|
||||||
|
"04020006, 04020027, 04020029, 04020030, 04020031",
|
||||||
|
"SN: 32090012, 32090014, 32090017, 32090018, 32090026, 32090021, 32090009, 32090010,",
|
||||||
|
"32090011, TR05617, 32090025, 32090013, 32090003, 302090005, 32090016, 32090023,",
|
||||||
|
"32090028, 32090008, 32090020, 32090019, 32090015, 32090024, 32090022, 302090006,",
|
||||||
|
"32090027, 32090029, 32090030, 32090031",
|
||||||
|
"Manufactured: 18/07/2024 – 17/03/2026",
|
||||||
|
"Dear Customer,",
|
||||||
|
]
|
||||||
|
summary = _parse_summary_kv(lines)
|
||||||
|
rows = _paired_rows_from_summary(summary)
|
||||||
|
self.assertEqual(len(rows), 28)
|
||||||
|
self.assertEqual(rows[9].udi, "04020001")
|
||||||
|
self.assertEqual(rows[9].serial_no, "TR05617")
|
||||||
|
|
||||||
|
|
||||||
|
class TestProseRejection(unittest.TestCase):
|
||||||
|
def test_detects_narrative_fragments(self):
|
||||||
|
self.assertTrue(_is_likely_prose("Dear Customer"))
|
||||||
|
self.assertTrue(
|
||||||
|
_is_likely_prose(
|
||||||
|
"The manufacturer has identified an error in the user manual."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertTrue(_is_likely_prose("Manufactured: 18/07/2024 – 17/03/2026"))
|
||||||
|
|
||||||
|
def test_clean_serial_rejects_prose(self):
|
||||||
|
self.assertEqual(_clean_serial_value("Dear Customer"), "")
|
||||||
|
self.assertEqual(_clean_serial_value("32090012"), "32090012")
|
||||||
|
|
||||||
|
def test_allows_long_comma_separated_identifier_lists(self):
|
||||||
|
line = (
|
||||||
|
"0012508285, 0012517045, 0012526102, 0012526103, 0012614786, 0012614787, "
|
||||||
|
"0012623740, 0012623741, 0012642959, 0012659419"
|
||||||
|
)
|
||||||
|
self.assertFalse(_is_likely_prose(line))
|
||||||
|
|
||||||
|
def test_clean_serial_keeps_comma_separated_lot_list(self):
|
||||||
|
value = "24B0729, 25B0770, 25H0171, 26B0542"
|
||||||
|
self.assertEqual(_clean_serial_value(value), value)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerialLabelParsing(unittest.TestCase):
|
||||||
|
def test_parse_sn_label_from_text(self):
|
||||||
|
lines = [
|
||||||
|
"UDI-DI: 01234567890123",
|
||||||
|
"SN: ABC12345",
|
||||||
|
"Date: 21-APR-2025",
|
||||||
|
]
|
||||||
|
summary = _parse_summary_kv(lines)
|
||||||
|
self.assertEqual(summary.get("udi"), "01234567890123")
|
||||||
|
self.assertEqual(summary.get("serial_no"), "ABC12345")
|
||||||
|
|
||||||
|
def test_sn_label_ignores_date_values(self):
|
||||||
|
lines = ["SN: 21-APR-2025"]
|
||||||
|
summary = _parse_summary_kv(lines)
|
||||||
|
self.assertNotIn("serial_no", summary)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
103
sfda_parser/sfda_scraper/weekly_list.py
Normal file
103
sfda_parser/sfda_scraper/weekly_list.py
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
"""Fetch the latest (newest) NCMDR weekly alert row from the SFDA website."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from sfda_parser.sfda_scraper.http_client import SFDA_BASE, create_session
|
||||||
|
|
||||||
|
WEEKLY_ALERT_URL = f"{SFDA_BASE}/en/weekly-alert"
|
||||||
|
|
||||||
|
# TEMPORARY: how many recent weekly bulletins to import/scan. Set to 1 for latest-only.
|
||||||
|
WEEKLY_BULLETINS_TO_IMPORT = 1
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class WeeklyAlertRow:
|
||||||
|
title: str
|
||||||
|
alert_date: date | None
|
||||||
|
pdf_url: str
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(value: str) -> date | None:
|
||||||
|
value = (value or "").strip()
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(value)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_table_row(tr) -> WeeklyAlertRow | None:
|
||||||
|
title_el = tr.select_one(".views-field-title")
|
||||||
|
date_el = tr.select_one(".views-field-field-date")
|
||||||
|
file_el = tr.select_one(".views-field-field-file a[href]")
|
||||||
|
|
||||||
|
if not title_el or not file_el:
|
||||||
|
return None
|
||||||
|
|
||||||
|
title = title_el.get_text(strip=True)
|
||||||
|
date_str = date_el.get_text(strip=True) if date_el else ""
|
||||||
|
href = file_el.get("href")
|
||||||
|
if not title or not href:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return WeeklyAlertRow(
|
||||||
|
title=title,
|
||||||
|
alert_date=_parse_date(date_str),
|
||||||
|
pdf_url=urljoin(SFDA_BASE, href),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_all_weekly_alerts(session=None) -> list[WeeklyAlertRow]:
|
||||||
|
"""Return every weekly bulletin row from the SFDA table (newest usually listed first)."""
|
||||||
|
session = session or create_session()
|
||||||
|
response = session.get(WEEKLY_ALERT_URL, timeout=60)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
rows: list[WeeklyAlertRow] = []
|
||||||
|
|
||||||
|
for tr in soup.select("tr"):
|
||||||
|
parsed = _parse_table_row(tr)
|
||||||
|
if parsed:
|
||||||
|
rows.append(parsed)
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
raise ValueError("Could not find any weekly alert rows on SFDA page")
|
||||||
|
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _sort_weekly_alerts_newest_first(rows: list[WeeklyAlertRow]) -> list[WeeklyAlertRow]:
|
||||||
|
"""Newest bulletin first; top-of-page wins when dates tie."""
|
||||||
|
indexed = list(enumerate(rows))
|
||||||
|
indexed.sort(
|
||||||
|
key=lambda item: (item[1].alert_date or date.min, -item[0]),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
return [item[1] for item in indexed]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_recent_weekly_alerts(
|
||||||
|
session=None, count: int | None = None
|
||||||
|
) -> list[WeeklyAlertRow]:
|
||||||
|
"""
|
||||||
|
Return the N newest weekly bulletins (default: WEEKLY_BULLETINS_TO_IMPORT).
|
||||||
|
|
||||||
|
Order: newest first. Use reversed() when importing oldest-to-newest.
|
||||||
|
"""
|
||||||
|
if count is None:
|
||||||
|
count = WEEKLY_BULLETINS_TO_IMPORT
|
||||||
|
rows = _sort_weekly_alerts_newest_first(fetch_all_weekly_alerts(session=session))
|
||||||
|
return rows[: max(1, count)]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_latest_weekly_alert(session=None) -> WeeklyAlertRow:
|
||||||
|
"""Return the single newest weekly bulletin."""
|
||||||
|
return fetch_recent_weekly_alerts(session=session, count=1)[0]
|
||||||
74
sfda_parser/sfda_scraper/weekly_pdf.py
Normal file
74
sfda_parser/sfda_scraper/weekly_pdf.py
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
"""Parse Safety Alerts table from the NCMDR weekly update PDF."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import pdfplumber
|
||||||
|
|
||||||
|
NCMDR_REF_PATTERN = re.compile(r"^SA-\d", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SafetyAlertRow:
|
||||||
|
ncmdr_ref: str
|
||||||
|
detail_url: str
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_refs_from_page(page) -> list[str]:
|
||||||
|
refs: list[str] = []
|
||||||
|
for table in page.extract_tables() or []:
|
||||||
|
for row in table or []:
|
||||||
|
if not row:
|
||||||
|
continue
|
||||||
|
for cell in row:
|
||||||
|
if not cell:
|
||||||
|
continue
|
||||||
|
text = str(cell).strip()
|
||||||
|
if NCMDR_REF_PATTERN.match(text):
|
||||||
|
refs.append(text)
|
||||||
|
return refs
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_links_from_page(page) -> list[str]:
|
||||||
|
hyperlinks = [
|
||||||
|
h
|
||||||
|
for h in (page.hyperlinks or [])
|
||||||
|
if h.get("uri") and "PublishDetails" in h["uri"]
|
||||||
|
]
|
||||||
|
hyperlinks.sort(key=lambda h: h.get("top", 0))
|
||||||
|
return [h["uri"] for h in hyperlinks]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_safety_alerts_from_pdf(pdf_bytes: bytes) -> list[SafetyAlertRow]:
|
||||||
|
alerts: list[SafetyAlertRow] = []
|
||||||
|
|
||||||
|
with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf:
|
||||||
|
for page in pdf.pages:
|
||||||
|
refs = _collect_refs_from_page(page)
|
||||||
|
links = _collect_links_from_page(page)
|
||||||
|
if not refs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if len(refs) != len(links):
|
||||||
|
# Fallback: zip min length and log mismatch in importer
|
||||||
|
pair_count = min(len(refs), len(links))
|
||||||
|
refs = refs[:pair_count]
|
||||||
|
links = links[:pair_count]
|
||||||
|
|
||||||
|
for ncmdr_ref, detail_url in zip(refs, links):
|
||||||
|
alerts.append(SafetyAlertRow(ncmdr_ref=ncmdr_ref, detail_url=detail_url))
|
||||||
|
|
||||||
|
# Deduplicate by NCMDR ref while preserving order
|
||||||
|
seen: set[str] = set()
|
||||||
|
unique: list[SafetyAlertRow] = []
|
||||||
|
for alert in alerts:
|
||||||
|
key = alert.ncmdr_ref.strip().upper()
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
unique.append(alert)
|
||||||
|
|
||||||
|
return unique
|
||||||
0
sfda_parser/templates/__init__.py
Normal file
0
sfda_parser/templates/__init__.py
Normal file
0
sfda_parser/templates/pages/__init__.py
Normal file
0
sfda_parser/templates/pages/__init__.py
Normal file
Loading…
x
Reference in New Issue
Block a user