mirror of
https://github.com/nicholaswilde/recipes.git
synced 2026-08-18 02:54:22 +00:00
feat(scripts): Implement scrape_to_cook.py and its unit tests
This commit is contained in:
nιcнolaѕ wιlde
Notes:
nιcнolaѕ wιlde
2026-06-14 00:22:38 -07:00
Task: Create scripts/scrape_to_cook.py & Write Unit Tests Summary of changes: Implemented scripts/scrape_to_cook.py to fetch and parse recipe pages (via JSON-LD schema or WPRM BeautifulSoup fallback). Added ISO 8601 duration parsing, CookLang ingredient formatting, and time range mapping to standard rules. Added unit tests covering all helpers and HTML scraping scenarios in scripts/test_scrape_to_cook.py, and updated pyproject.toml dependencies to include beautifulsoup4 and pytest. Created/modified files: - scripts/scrape_to_cook.py - scripts/test_scrape_to_cook.py - pyproject.toml - uv.lock Why: To automate the ingestion of recipe webpages and compile them directly to CookLang formats, saving significant import time.
@ -8,4 +8,6 @@ dependencies = [
|
||||
"zensical>=0.0.32",
|
||||
"pyyaml>=6.0.1",
|
||||
"pillow>=12.2.0",
|
||||
"beautifulsoup4>=4.12.0",
|
||||
"pytest>=8.0.0",
|
||||
]
|
||||
|
||||
459
scripts/scrape_to_cook.py
Normal file
459
scripts/scrape_to_cook.py
Normal file
@ -0,0 +1,459 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import re
|
||||
import os
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from bs4 import BeautifulSoup
|
||||
import yaml
|
||||
|
||||
COMMON_UNITS = {
|
||||
"cup", "cups", "tbsp", "tablespoon", "tablespoons", "tsp", "teaspoon", "teaspoons",
|
||||
"g", "gram", "grams", "ml", "milliliter", "milliliters", "oz", "ounce", "ounces",
|
||||
"lb", "pound", "pounds", "kg", "kilogram", "kilograms", "can", "cans", "clove",
|
||||
"cloves", "pinch", "pinches", "slice", "slices", "package", "packages", "bag",
|
||||
"bags", "canister", "canisters", "jar", "jars", "head", "heads", "bunch", "bunches",
|
||||
"sprig", "sprigs", "piece", "pieces", "large", "medium", "small", "handful", "handfuls"
|
||||
}
|
||||
|
||||
COMMON_COOKWARE_KEYWORDS = [
|
||||
"bowl", "mixing bowl", "pan", "baking pan", "baking sheet", "saucepan", "pot",
|
||||
"skillet", "frying pan", "griddle", "oven", "microwave", "blender", "food processor",
|
||||
"stand mixer", "hand mixer", "whisk", "spatula", "wooden spoon", "knife", "cutting board",
|
||||
"strainer", "colander", "peeler", "grater", "measuring cup", "measuring spoon",
|
||||
"casserole dish", "tart pan", "muffin tin", "loaf pan", "cookie scoop", "wire rack"
|
||||
]
|
||||
|
||||
def parse_iso_duration(duration_str):
|
||||
if not duration_str:
|
||||
return ""
|
||||
m = re.match(r'PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?', duration_str)
|
||||
if not m:
|
||||
return duration_str
|
||||
hours, minutes, seconds = m.groups()
|
||||
parts = []
|
||||
if hours:
|
||||
h_val = int(hours)
|
||||
parts.append(f"{h_val} hour" + ("s" if h_val != 1 else ""))
|
||||
if minutes:
|
||||
m_val = int(minutes)
|
||||
parts.append(f"{m_val} minute" + ("s" if m_val != 1 else ""))
|
||||
return " ".join(parts)
|
||||
|
||||
def parse_fraction(val_str):
|
||||
val_str = val_str.strip()
|
||||
unicode_fractions = {
|
||||
'½': 0.5, '⅓': 0.33, '⅔': 0.67, '¼': 0.25, '¾': 0.75,
|
||||
'⅕': 0.2, '⅖': 0.4, '⅗': 0.6, '⅘': 0.8, '⅙': 0.17, '⅚': 0.83,
|
||||
'⅛': 0.125, '⅜': 0.375, '⅝': 0.625, '⅞': 0.875
|
||||
}
|
||||
for char, float_val in unicode_fractions.items():
|
||||
if char in val_str:
|
||||
val_str = val_str.replace(char, f" {float_val}")
|
||||
|
||||
parts = val_str.split()
|
||||
total = 0.0
|
||||
for part in parts:
|
||||
if '/' in part:
|
||||
try:
|
||||
num, denom = part.split('/')
|
||||
total += float(num) / float(denom)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
total += float(part)
|
||||
except ValueError:
|
||||
pass
|
||||
if total > 0.0:
|
||||
if total.is_integer():
|
||||
return str(int(total))
|
||||
return f"{total:.2f}".rstrip('0').rstrip('.')
|
||||
return val_str
|
||||
|
||||
def parse_ingredient(ing_line):
|
||||
ignored_ingredients = []
|
||||
try:
|
||||
config_path = os.path.join("cook", "config", "ignored_ingredients.yaml")
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
ignored_ingredients = yaml.safe_load(f) or []
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ing_line_clean = ing_line.strip()
|
||||
for ignored in ignored_ingredients:
|
||||
if ing_line_clean.lower() == ignored.lower():
|
||||
return ing_line_clean
|
||||
|
||||
qty_regex = r'^(\d+(?:\s+\d+/\d+|\s+[½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞])?|\d+/\d+|\d+\.\d+|[½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞])'
|
||||
qty_match = re.match(qty_regex, ing_line_clean)
|
||||
|
||||
if not qty_match:
|
||||
return f"@{ing_line_clean}{{}}"
|
||||
|
||||
qty_raw = qty_match.group(1)
|
||||
remaining = ing_line_clean[len(qty_raw):].strip()
|
||||
qty = parse_fraction(qty_raw)
|
||||
|
||||
words = remaining.split()
|
||||
if not words:
|
||||
return f"@ingredient{{{qty}}}"
|
||||
|
||||
first_word = words[0].lower().rstrip(',')
|
||||
unit = ""
|
||||
ing_name_words = words
|
||||
if first_word in COMMON_UNITS:
|
||||
unit = first_word
|
||||
ing_name_words = words[1:]
|
||||
|
||||
if ing_name_words and ing_name_words[0].lower() == "of":
|
||||
ing_name_words = ing_name_words[1:]
|
||||
|
||||
name = " ".join(ing_name_words).strip()
|
||||
|
||||
notes = ""
|
||||
m_notes = re.search(r'\(([^)]+)\)$', name)
|
||||
if m_notes:
|
||||
notes = m_notes.group(1).strip()
|
||||
name = name[:m_notes.start()].strip()
|
||||
|
||||
name = name.rstrip(',').strip()
|
||||
|
||||
for ignored in ignored_ingredients:
|
||||
if name.lower() == ignored.lower():
|
||||
return ing_line_clean
|
||||
|
||||
unit_part = f"%{unit}" if unit else ""
|
||||
notes_part = f" ({notes})" if notes else ""
|
||||
|
||||
return f"@{name}{{{qty}{unit_part}}}{notes_part}"
|
||||
|
||||
def format_time_range(text):
|
||||
pattern_range = r'(\d+)\s*(?:-|to)\s*(\d+)\s*(minutes|minute|hours|hour|seconds|second|secs|sec)'
|
||||
m_range = re.search(pattern_range, text, re.IGNORECASE)
|
||||
if m_range:
|
||||
num1, num2, unit = m_range.groups()
|
||||
replacement = f"{num1} to ~{{{num2}%{unit}}}"
|
||||
return re.sub(pattern_range, replacement, text, flags=re.IGNORECASE)
|
||||
|
||||
pattern_single = r'(about|approx|approximately)\s*(\d+)\s*(minutes|minute|hours|hour|seconds|second|secs|sec)'
|
||||
m_single = re.search(pattern_single, text, re.IGNORECASE)
|
||||
if m_single:
|
||||
prefix, num, unit = m_single.groups()
|
||||
replacement = f"{prefix} ~{{{num}%{unit}}}"
|
||||
return re.sub(pattern_single, replacement, text, flags=re.IGNORECASE)
|
||||
|
||||
return text
|
||||
|
||||
def resolve_category(category_name):
|
||||
if not category_name:
|
||||
return "main"
|
||||
category_lower = category_name.lower().strip()
|
||||
|
||||
if any(kw in category_lower for kw in ["dessert", "sweet", "cake", "cookie", "bar"]):
|
||||
if any(kw in category_lower for kw in ["cookie", "bar"]):
|
||||
return "cookies-and-bars"
|
||||
return "desserts"
|
||||
if any(kw in category_lower for kw in ["bread", "bun", "dough", "roll"]):
|
||||
return "breads"
|
||||
if any(kw in category_lower for kw in ["breakfast", "brunch", "waffle", "pancake"]):
|
||||
return "breakfast"
|
||||
if any(kw in category_lower for kw in ["drink", "beverage", "cocktail", "smoothie"]):
|
||||
return "beverages"
|
||||
if any(kw in category_lower for kw in ["sauce", "dressing", "dip", "salsa", "gravy"]):
|
||||
return "sauces-and-dressings"
|
||||
if any(kw in category_lower for kw in ["soup", "stew", "chowder", "chili"]):
|
||||
return "soups-and-stews"
|
||||
if any(kw in category_lower for kw in ["salad", "side", "appetizer", "snack"]):
|
||||
return "sides"
|
||||
if any(kw in category_lower for kw in ["lunch"]):
|
||||
return "lunches"
|
||||
if any(kw in category_lower for kw in ["mediterranean"]):
|
||||
return "mediterranean"
|
||||
|
||||
return "main"
|
||||
|
||||
def fetch_html(url):
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req) as response:
|
||||
return response.read().decode('utf-8')
|
||||
|
||||
def find_recipe_in_json(data):
|
||||
if isinstance(data, dict):
|
||||
if data.get("@type") == "Recipe" or data.get("@type") == ["Recipe"]:
|
||||
return data
|
||||
for k, v in data.items():
|
||||
if k == "@graph" and isinstance(v, list):
|
||||
for item in v:
|
||||
recipe = find_recipe_in_json(item)
|
||||
if recipe:
|
||||
return recipe
|
||||
elif isinstance(v, (dict, list)):
|
||||
recipe = find_recipe_in_json(v)
|
||||
if recipe:
|
||||
return recipe
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
recipe = find_recipe_in_json(item)
|
||||
if recipe:
|
||||
return recipe
|
||||
return None
|
||||
|
||||
def extract_json_ld_recipe(html):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
scripts = soup.find_all("script", type="application/ld+json")
|
||||
for script in scripts:
|
||||
try:
|
||||
data = json.loads(script.string or "")
|
||||
recipe = find_recipe_in_json(data)
|
||||
if recipe:
|
||||
return recipe
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
def parse_instructions(instructions_data):
|
||||
steps = []
|
||||
if isinstance(instructions_data, str):
|
||||
steps.append(instructions_data)
|
||||
elif isinstance(instructions_data, list):
|
||||
for item in instructions_data:
|
||||
if isinstance(item, str):
|
||||
steps.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if item.get("@type") == "HowToSection":
|
||||
elements = item.get("itemListElement") or []
|
||||
for elem in elements:
|
||||
if isinstance(elem, dict) and elem.get("@type") == "HowToStep":
|
||||
steps.append(elem.get("text") or elem.get("name") or "")
|
||||
elif item.get("@type") == "HowToStep":
|
||||
steps.append(item.get("text") or item.get("name") or "")
|
||||
elif "text" in item:
|
||||
steps.append(item.get("text") or "")
|
||||
return [s.strip() for s in steps if s.strip()]
|
||||
|
||||
def extract_wprm_recipe(html):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
recipe_container = soup.find(class_="wprm-recipe-container")
|
||||
if not recipe_container:
|
||||
return None
|
||||
|
||||
recipe = {}
|
||||
name_el = recipe_container.find(class_="wprm-recipe-name")
|
||||
recipe["name"] = name_el.get_text(strip=True) if name_el else "Unknown Recipe"
|
||||
|
||||
servings_el = recipe_container.find(class_="wprm-recipe-servings")
|
||||
recipe["servings"] = servings_el.get_text(strip=True) if servings_el else ""
|
||||
|
||||
prep_el = recipe_container.find(class_="wprm-recipe-prep_time")
|
||||
recipe["prep_time"] = prep_el.get_text(strip=True) if prep_el else ""
|
||||
cook_el = recipe_container.find(class_="wprm-recipe-cook_time")
|
||||
recipe["cook_time"] = cook_el.get_text(strip=True) if cook_el else ""
|
||||
total_el = recipe_container.find(class_="wprm-recipe-total_time")
|
||||
recipe["total_time"] = total_el.get_text(strip=True) if total_el else ""
|
||||
|
||||
image_el = recipe_container.find("img")
|
||||
image_url = ""
|
||||
if image_el:
|
||||
image_url = image_el.get("data-lazy-src") or image_el.get("src") or ""
|
||||
recipe["image_url"] = image_url
|
||||
|
||||
ingredients = []
|
||||
ing_elements = recipe_container.find_all(class_="wprm-recipe-ingredient")
|
||||
for ing in ing_elements:
|
||||
amount_el = ing.find(class_="wprm-recipe-ingredient-amount")
|
||||
unit_el = ing.find(class_="wprm-recipe-ingredient-unit")
|
||||
name_el = ing.find(class_="wprm-recipe-ingredient-name")
|
||||
notes_el = ing.find(class_="wprm-recipe-ingredient-notes")
|
||||
|
||||
amount = amount_el.get_text(strip=True) if amount_el else ""
|
||||
unit = unit_el.get_text(strip=True) if unit_el else ""
|
||||
name = name_el.get_text(strip=True) if name_el else ""
|
||||
notes = notes_el.get_text(strip=True) if notes_el else ""
|
||||
|
||||
full_ing = f"{amount} {unit} {name}".replace(" ", " ").strip()
|
||||
if notes:
|
||||
full_ing += f" ({notes})"
|
||||
ingredients.append(full_ing)
|
||||
recipe["ingredients"] = ingredients
|
||||
|
||||
instructions = []
|
||||
inst_elements = recipe_container.find_all(class_="wprm-recipe-instruction")
|
||||
for inst in inst_elements:
|
||||
text_el = inst.find(class_="wprm-recipe-instruction-text")
|
||||
text = text_el.get_text(strip=True) if text_el else inst.get_text(strip=True)
|
||||
instructions.append(text)
|
||||
recipe["instructions"] = instructions
|
||||
|
||||
return recipe
|
||||
|
||||
def download_image(url, output_path):
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req) as response:
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.read())
|
||||
|
||||
def tag_cookware(step_text):
|
||||
keywords = sorted(COMMON_COOKWARE_KEYWORDS, key=len, reverse=True)
|
||||
for kw in keywords:
|
||||
pattern = rf'(?<![#@])\b{re.escape(kw)}\b'
|
||||
step_text = re.sub(pattern, f"#{kw}{{}}", step_text, flags=re.IGNORECASE)
|
||||
return step_text
|
||||
|
||||
def compile_cooklang(recipe_data, source_url):
|
||||
lines = []
|
||||
lines.append(f">> source: {source_url}")
|
||||
|
||||
servings = recipe_data.get("servings")
|
||||
if servings:
|
||||
lines.append(f">> serves: {servings}")
|
||||
|
||||
prep_time = recipe_data.get("prep_time")
|
||||
if prep_time:
|
||||
lines.append(f">> prep time: {prep_time}")
|
||||
cook_time = recipe_data.get("cook_time")
|
||||
if cook_time:
|
||||
lines.append(f">> cook time: {cook_time}")
|
||||
total_time = recipe_data.get("total_time")
|
||||
if total_time:
|
||||
lines.append(f">> total time: {total_time}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Declarations of all ingredients at the beginning (commented out)
|
||||
parsed_ingredients = []
|
||||
raw_ing_names = []
|
||||
for ing in recipe_data.get("ingredients", []):
|
||||
parsed = parse_ingredient(ing)
|
||||
parsed_ingredients.append(parsed)
|
||||
# Extract name from @name{...} format to tag in steps
|
||||
m_name = re.match(r'^@([^{]+)\{', parsed)
|
||||
if m_name:
|
||||
raw_ing_names.append(m_name.group(1))
|
||||
|
||||
for parsed in parsed_ingredients:
|
||||
lines.append(parsed)
|
||||
|
||||
lines.append("")
|
||||
|
||||
# Instruction steps
|
||||
# Sort names by length descending to replace longest names first
|
||||
raw_ing_names = sorted(list(set(raw_ing_names)), key=len, reverse=True)
|
||||
|
||||
for step in recipe_data.get("instructions", []):
|
||||
step_clean = format_time_range(step)
|
||||
step_clean = tag_cookware(step_clean)
|
||||
|
||||
# Tag ingredients in steps
|
||||
for ing_name in raw_ing_names:
|
||||
pattern = rf'(?<![#@])\b{re.escape(ing_name)}\b'
|
||||
step_clean = re.sub(pattern, f"@{ing_name}", step_clean, flags=re.IGNORECASE)
|
||||
|
||||
lines.append(step_clean)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Scrape a recipe from a URL and save as CookLang (.cook) and image.")
|
||||
parser.add_argument("url", help="Recipe webpage URL")
|
||||
parser.add_argument("-c", "--category", help="Category folder override")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Fetching {args.url}...")
|
||||
html = fetch_html(args.url)
|
||||
|
||||
# 1. Try JSON-LD Recipe
|
||||
recipe_data = None
|
||||
recipe_json = extract_json_ld_recipe(html)
|
||||
if recipe_json:
|
||||
recipe_data = {}
|
||||
recipe_data["name"] = recipe_json.get("name") or "Unknown Recipe"
|
||||
|
||||
yield_val = recipe_json.get("recipeYield")
|
||||
if isinstance(yield_val, list) and yield_val:
|
||||
yield_val = yield_val[0]
|
||||
servings = ""
|
||||
if yield_val:
|
||||
m = re.search(r'\d+', str(yield_val))
|
||||
if m:
|
||||
servings = m.group(0)
|
||||
recipe_data["servings"] = servings
|
||||
|
||||
recipe_data["prep_time"] = parse_iso_duration(recipe_json.get("prepTime"))
|
||||
recipe_data["cook_time"] = parse_iso_duration(recipe_json.get("cookTime"))
|
||||
recipe_data["total_time"] = parse_iso_duration(recipe_json.get("totalTime"))
|
||||
|
||||
img_data = recipe_json.get("image")
|
||||
image_url = ""
|
||||
if isinstance(img_data, list) and img_data:
|
||||
image_url = img_data[0]
|
||||
elif isinstance(img_data, dict):
|
||||
image_url = img_data.get("url") or ""
|
||||
elif isinstance(img_data, str):
|
||||
image_url = img_data
|
||||
if image_url:
|
||||
recipe_data["image_url"] = urllib.parse.urljoin(args.url, image_url)
|
||||
else:
|
||||
recipe_data["image_url"] = ""
|
||||
|
||||
recipe_data["ingredients"] = recipe_json.get("recipeIngredient") or []
|
||||
recipe_data["instructions"] = parse_instructions(recipe_json.get("recipeInstructions"))
|
||||
|
||||
category_json = recipe_json.get("recipeCategory")
|
||||
if isinstance(category_json, list) and category_json:
|
||||
category_json = category_json[0]
|
||||
recipe_data["category"] = str(category_json or "")
|
||||
print("Successfully parsed recipe via JSON-LD Schema.")
|
||||
|
||||
if not recipe_data or not recipe_data.get("ingredients"):
|
||||
# 2. Try Fallback WPRM BeautifulSoup
|
||||
print("JSON-LD parse did not return a valid recipe. Trying WPRM BeautifulSoup fallback...")
|
||||
recipe_data = extract_wprm_recipe(html)
|
||||
|
||||
if not recipe_data or not recipe_data.get("ingredients"):
|
||||
print("Error: Could not parse recipe from the webpage.")
|
||||
sys.exit(1)
|
||||
|
||||
# Resolve category
|
||||
category = args.category
|
||||
if not category:
|
||||
category = resolve_category(recipe_data.get("category") or recipe_data.get("name"))
|
||||
|
||||
# Format Title/Filename (clean special characters)
|
||||
title = recipe_data.get("name")
|
||||
filename = "".join(c for c in title if c.isalnum() or c in " -_").strip()
|
||||
|
||||
# Directories
|
||||
target_dir = os.path.join("cook", category)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
cook_path = os.path.join(target_dir, f"{filename}.cook")
|
||||
jpg_path = os.path.join(target_dir, f"{filename}.jpg")
|
||||
|
||||
# Save Cooklang file
|
||||
cook_content = compile_cooklang(recipe_data, args.url)
|
||||
with open(cook_path, "w", encoding="utf-8") as f:
|
||||
f.write(cook_content)
|
||||
print(f"Saved Cooklang recipe to {cook_path}")
|
||||
|
||||
# Save Image
|
||||
image_url = recipe_data.get("image_url")
|
||||
if image_url:
|
||||
try:
|
||||
print(f"Downloading recipe image from {image_url}...")
|
||||
download_image(image_url, jpg_path)
|
||||
print(f"Saved image to {jpg_path}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not download image: {e}")
|
||||
|
||||
print("Recipe import complete!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
126
scripts/test_scrape_to_cook.py
Normal file
126
scripts/test_scrape_to_cook.py
Normal file
@ -0,0 +1,126 @@
|
||||
import unittest
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Import functions from scrape_to_cook directly
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("scrape_to_cook", "scripts/scrape_to_cook.py")
|
||||
scrape_to_cook = importlib.util.module_from_spec(spec)
|
||||
sys.modules["scrape_to_cook"] = scrape_to_cook
|
||||
spec.loader.exec_module(scrape_to_cook)
|
||||
|
||||
class TestScrapeToCook(unittest.TestCase):
|
||||
def test_module_exists(self):
|
||||
self.assertIsNotNone(scrape_to_cook, "scrape_to_cook module should exist")
|
||||
|
||||
def test_parse_iso_duration(self):
|
||||
self.assertEqual(scrape_to_cook.parse_iso_duration("PT15M"), "15 minutes")
|
||||
self.assertEqual(scrape_to_cook.parse_iso_duration("PT1H30M"), "1 hour 30 minutes")
|
||||
self.assertEqual(scrape_to_cook.parse_iso_duration("PT2H"), "2 hours")
|
||||
self.assertEqual(scrape_to_cook.parse_iso_duration(""), "")
|
||||
|
||||
def test_parse_ingredient(self):
|
||||
self.assertEqual(
|
||||
scrape_to_cook.parse_ingredient("2 1/2 cups all-purpose flour"),
|
||||
"@all-purpose flour{2.5%cups}"
|
||||
)
|
||||
self.assertEqual(
|
||||
scrape_to_cook.parse_ingredient("1 tsp salt"),
|
||||
"@salt{1%tsp}"
|
||||
)
|
||||
# Ignored ingredient from cook/config/ignored_ingredients.yaml
|
||||
self.assertEqual(
|
||||
scrape_to_cook.parse_ingredient("nonstick baking spray"),
|
||||
"nonstick baking spray"
|
||||
)
|
||||
self.assertEqual(
|
||||
scrape_to_cook.parse_ingredient("3 large eggs"),
|
||||
"@eggs{3%large}"
|
||||
)
|
||||
|
||||
def test_format_time_range(self):
|
||||
self.assertEqual(
|
||||
scrape_to_cook.format_time_range("7-8 minutes"),
|
||||
"7 to ~{8%minutes}"
|
||||
)
|
||||
self.assertEqual(
|
||||
scrape_to_cook.format_time_range("10 to 12 minutes"),
|
||||
"10 to ~{12%minutes}"
|
||||
)
|
||||
self.assertEqual(
|
||||
scrape_to_cook.format_time_range("about 5 minutes"),
|
||||
"about ~{5%minutes}"
|
||||
)
|
||||
|
||||
def test_resolve_category(self):
|
||||
self.assertEqual(scrape_to_cook.resolve_category("Dessert"), "desserts")
|
||||
self.assertEqual(scrape_to_cook.resolve_category("Bread & Buns"), "breads")
|
||||
self.assertEqual(scrape_to_cook.resolve_category("Soup"), "soups-and-stews")
|
||||
self.assertEqual(scrape_to_cook.resolve_category("Something Unknown"), "main")
|
||||
|
||||
def test_extract_json_ld_recipe(self):
|
||||
html = """
|
||||
<html>
|
||||
<head>
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@graph": [
|
||||
{
|
||||
"@type": "Recipe",
|
||||
"name": "Dummy Chocolate Cake",
|
||||
"recipeYield": ["8 servings"],
|
||||
"prepTime": "PT15M",
|
||||
"cookTime": "PT45M",
|
||||
"recipeIngredient": [
|
||||
"1 cup sugar",
|
||||
"2 cups flour"
|
||||
],
|
||||
"recipeInstructions": [
|
||||
{
|
||||
"@type": "HowToStep",
|
||||
"text": "Mix sugar and flour."
|
||||
},
|
||||
{
|
||||
"@type": "HowToStep",
|
||||
"text": "Bake for 45 minutes."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body></body>
|
||||
</html>
|
||||
"""
|
||||
recipe = scrape_to_cook.extract_json_ld_recipe(html)
|
||||
self.assertIsNotNone(recipe)
|
||||
self.assertEqual(recipe.get("name"), "Dummy Chocolate Cake")
|
||||
self.assertEqual(recipe.get("recipeYield"), ["8 servings"])
|
||||
self.assertEqual(recipe.get("recipeIngredient"), ["1 cup sugar", "2 cups flour"])
|
||||
|
||||
def test_extract_wprm_recipe(self):
|
||||
html = """
|
||||
<div class="wprm-recipe-container">
|
||||
<span class="wprm-recipe-name">WPRM Oatmeal Cookies</span>
|
||||
<span class="wprm-recipe-servings">12</span>
|
||||
<div class="wprm-recipe-ingredient">
|
||||
<span class="wprm-recipe-ingredient-amount">1</span>
|
||||
<span class="wprm-recipe-ingredient-unit">cup</span>
|
||||
<span class="wprm-recipe-ingredient-name">oats</span>
|
||||
</div>
|
||||
<div class="wprm-recipe-instruction">
|
||||
<div class="wprm-recipe-instruction-text">Bake cookies.</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
recipe = scrape_to_cook.extract_wprm_recipe(html)
|
||||
self.assertIsNotNone(recipe)
|
||||
self.assertEqual(recipe["name"], "WPRM Oatmeal Cookies")
|
||||
self.assertEqual(recipe["servings"], "12")
|
||||
self.assertEqual(recipe["ingredients"], ["1 cup oats"])
|
||||
self.assertEqual(recipe["instructions"], ["Bake cookies."])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
78
uv.lock
generated
78
uv.lock
generated
@ -2,6 +2,19 @@ version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.14"
|
||||
|
||||
[[package]]
|
||||
name = "beautifulsoup4"
|
||||
version = "4.15.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "soupsieve" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.3.1"
|
||||
@ -32,6 +45,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2d/82/e5d2c1c67d19841e9edc74954c827444ae826978499bde3dfc1d007c8c11/deepmerge-2.0-py3-none-any.whl", hash = "sha256:6de9ce507115cff0bed95ff0ce9ecc31088ef50cbdf09bc90a09349a318b3d00", size = 13475, upload-time = "2024-08-30T05:31:48.659Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jinja2"
|
||||
version = "3.1.6"
|
||||
@ -83,6 +105,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pillow"
|
||||
version = "12.2.0"
|
||||
@ -116,6 +147,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.20.0"
|
||||
@ -138,6 +178,22 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/85/545a951eecc270fcd688288c600017e2050a1aacb56c711d208586d3e470/pymdown_extensions-10.21.3-py3-none-any.whl", hash = "sha256:d7a5d08014fc571e80ca21dd6f854e31f94c489800350564d55d15b3c41e76b6", size = 269002, upload-time = "2026-05-13T12:57:30.296Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0.3"
|
||||
@ -169,18 +225,31 @@ name = "recipes"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "pillow" },
|
||||
{ name = "pytest" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "zensical" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "beautifulsoup4", specifier = ">=4.12.0" },
|
||||
{ name = "pillow", specifier = ">=12.2.0" },
|
||||
{ name = "pytest", specifier = ">=8.0.0" },
|
||||
{ name = "pyyaml", specifier = ">=6.0.1" },
|
||||
{ name = "zensical", specifier = ">=0.0.32" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.8.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tomli"
|
||||
version = "2.4.1"
|
||||
@ -208,6 +277,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.15.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zensical"
|
||||
version = "0.0.43"
|
||||
|
||||
Reference in New Issue
Block a user