Add category search, license handling, and message page improvements

Add /category route to find Wikipedia articles without images in a
category using the MediaWiki API. Filter out non-content images (UI
icons, logos) when checking articles.

Show image license on message page with alternate message for non-free
CC licenses (NC/ND) explaining Wikipedia's restrictions. For photos
with free licenses, show upload options linking to UploadWizard instead
of a message form.

Add Flickr CC 4.0 license codes, user profile links, previous message
detection from sent mail index, and back-navigation between category,
search results, and message pages.

Closes #3

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

main.py

@@ -34,17 +34,20 @@ COMMONS_CACHE_FILE = (
     Path(__file__).parent / "commons_contributions" / "thumbnail_cache.json"
 )
 SENT_MAIL_DIR = Path(__file__).parent / "sent_mail" / "messages"
+SENT_MAIL_INDEX_FILE = Path(__file__).parent / "sent_mail" / "messages_index.json"
 SENT_MAIL_INDEX_CACHE = (
     Path(__file__).parent / "commons_contributions" / "sent_mail_index.json"
 )
 COMMONS_CACHE_MAX_AGE = 86400 * 7  # Cache for 7 days
 RECENT_UPLOADS_COUNT = 24
 
-# User agent for Commons API requests
+# User agent for Wikimedia API requests
-COMMONS_USER_AGENT = (
+WIKIMEDIA_USER_AGENT = (
     "FlickrMail/1.0 (https://edwardbetts.com/flickr_mail/; edward@4angle.com)"
 )
 
+WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"
+
 # Browser-like headers for Flickr requests
 BROWSER_HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -64,18 +67,26 @@ BROWSER_HEADERS = {
 # Flickr license codes to human-readable names
 FLICKR_LICENSES = {
     0: "All Rights Reserved",
-    1: "CC BY-NC-SA",
+    1: "CC BY-NC-SA 2.0",
-    2: "CC BY-NC",
+    2: "CC BY-NC 2.0",
-    3: "CC BY-NC-ND",
+    3: "CC BY-NC-ND 2.0",
-    4: "CC BY",
+    4: "CC BY 2.0",
-    5: "CC BY-SA",
+    5: "CC BY-SA 2.0",
-    6: "CC BY-ND",
+    6: "CC BY-ND 2.0",
     7: "No known copyright",
     8: "US Government",
     9: "CC0",
     10: "Public Domain",
+    # CC 4.0 licenses (codes confirmed via Flickr)
+    16: "CC BY-NC-ND 4.0",
 }
 
+# Non-free CC licenses (NC or ND restrictions)
+NONFREE_CC_LICENSES = {1, 2, 3, 6, 11, 12, 13, 16}
+
+# Wikipedia-compatible free licenses
+FREE_LICENSES = {4, 5, 7, 8, 9, 10, 14, 15}
+
 
 PHOTOS_PER_PAGE = 25
 
@@ -296,7 +307,7 @@ def fetch_commons_thumbnails(titles: list[str]) -> dict[str, str]:
         "format": "json",
     }
 
-    headers = {"User-Agent": COMMONS_USER_AGENT}
+    headers = {"User-Agent": WIKIMEDIA_USER_AGENT}
 
     try:
         response = requests.get(
@@ -404,6 +415,165 @@ def get_recent_commons_uploads() -> tuple[list[CommonsUpload], int]:
     return result, total_matched
 
 
+def get_previous_messages(flickr_user: str, flickr_username: str) -> list[dict]:
+    """Get previous messages sent to a Flickr user.
+
+    Checks both the display name (flickr_user) and username (flickr_username)
+    against the recipient field in the messages index.
+    """
+    if not SENT_MAIL_INDEX_FILE.exists():
+        return []
+
+    try:
+        with open(SENT_MAIL_INDEX_FILE) as f:
+            messages = json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return []
+
+    # Normalize for case-insensitive comparison
+    flickr_user_lower = flickr_user.lower() if flickr_user else ""
+    flickr_username_lower = flickr_username.lower() if flickr_username else ""
+
+    matches = []
+    for msg in messages:
+        recipient = msg.get("recipient", "").lower()
+        if recipient and (recipient == flickr_user_lower or recipient == flickr_username_lower):
+            matches.append(msg)
+
+    return matches
+
+
+def parse_category_input(category_input: str) -> str | None:
+    """Parse category title from URL or direct input.
+
+    Returns the category title with 'Category:' prefix, or None if invalid.
+    """
+    category_input = category_input.strip()
+
+    # Handle URL format: https://en.wikipedia.org/wiki/Category:Example
+    if "wikipedia.org" in category_input:
+        match = re.search(r"/wiki/(Category:[^#?]+)", category_input)
+        if match:
+            return unquote(match.group(1)).replace("_", " ")
+        return None
+
+    # Handle direct input - add Category: prefix if missing
+    if category_input.startswith("Category:"):
+        return category_input.replace("_", " ")
+
+    # Assume it's just the category name
+    return f"Category:{category_input.replace('_', ' ')}"
+
+
+@dataclasses.dataclass
+class ArticleWithoutImage:
+    """Represents a Wikipedia article that needs an image."""
+
+    title: str
+    pageid: int
+
+    @property
+    def wikipedia_url(self) -> str:
+        """URL to the Wikipedia article."""
+        return f"https://en.wikipedia.org/wiki/{self.title.replace(' ', '_')}"
+
+    @property
+    def search_url(self) -> str:
+        """URL to search for this article in Flickr Mail."""
+        return f"/?enwp={quote(self.title)}"
+
+
+# Common non-content images to ignore when checking if an article has images
+NON_CONTENT_IMAGE_PATTERNS = [
+    "OOjs UI icon",
+    "Commons-logo",
+    "Symbol ",
+    "Edit-ltr",
+    "Ambox ",
+    "Question book",
+    "Wiki letter",
+    "Text document",
+    "Folder ",
+    "Crystal ",
+    "Nuvola ",
+    "Gnome-",
+    "Disambig ",
+    "DAB ",
+]
+
+
+def has_content_image(images: list[dict]) -> bool:
+    """Check if an article has a content image (not just UI icons/logos)."""
+    for img in images:
+        title = img.get("title", "")
+        # Skip if it matches any non-content pattern
+        is_non_content = any(pattern in title for pattern in NON_CONTENT_IMAGE_PATTERNS)
+        if not is_non_content:
+            return True
+    return False
+
+
+def get_articles_without_images(
+    category: str, limit: int = 100
+) -> tuple[list[ArticleWithoutImage], str | None]:
+    """Get articles in a category that don't have images.
+
+    Uses generator=categorymembers with prop=images to efficiently check
+    multiple articles in a single API request.
+
+    Returns a tuple of (articles_list, continue_token).
+    """
+    params = {
+        "action": "query",
+        "generator": "categorymembers",
+        "gcmtitle": category,
+        "gcmtype": "page",  # Only articles, not subcategories or files
+        "gcmnamespace": "0",  # Main namespace only
+        "gcmlimit": str(limit),
+        "prop": "images",
+        "imlimit": "max",  # Need enough to check all pages in batch
+        "format": "json",
+    }
+
+    headers = {"User-Agent": WIKIMEDIA_USER_AGENT}
+
+    try:
+        response = requests.get(
+            WIKIPEDIA_API, params=params, headers=headers, timeout=30
+        )
+        response.raise_for_status()
+        data = response.json()
+    except (requests.RequestException, json.JSONDecodeError) as e:
+        print(f"Wikipedia API error: {e}")
+        return [], None
+
+    articles_without_images: list[ArticleWithoutImage] = []
+
+    pages = data.get("query", {}).get("pages", {})
+    for page in pages.values():
+        images = page.get("images", [])
+
+        # Skip if page has content images (not just UI icons)
+        if has_content_image(images):
+            continue
+
+        title = page.get("title", "")
+        pageid = page.get("pageid", 0)
+
+        if title and pageid:
+            articles_without_images.append(
+                ArticleWithoutImage(title=title, pageid=pageid)
+            )
+
+    # Sort by title for consistent display
+    articles_without_images.sort(key=lambda a: a.title)
+
+    # Get continue token if there are more results
+    continue_token = data.get("continue", {}).get("gcmcontinue")
+
+    return articles_without_images, continue_token
+
+
 def is_valid_flickr_image_url(url: str) -> bool:
     """Check if URL is a valid Flickr static image URL."""
     valid_prefixes = (
@@ -608,9 +778,13 @@ def start() -> str:
         wiki_part2 = name.replace(" ", "_")
         wikipedia_url = wiki_part1 + wiki_part2
 
-    if "_(" in name:
+    # Remove disambiguation suffix like "(academic)" for Flickr search
-        name = name[: name.find("_(")]
     name = name.replace("_", " ")
+    if " (" in name:
+        name = name[: name.find(" (")]
+
+    # Get category param if coming from category search
+    cat = flask.request.args.get("cat")
 
     flickr_url = flask.request.args.get("flickr")
     if not flickr_url:
@@ -623,6 +797,7 @@ def start() -> str:
             name=name,
             enwp=enwp,
             search_result=search_result,
+            cat=cat,
         )
 
     if "/in/" in flickr_url:
@@ -644,6 +819,40 @@ def start() -> str:
     if img_url and not is_valid_flickr_image_url(img_url):
         img_url = None
 
+    # Get flickr_user name and build profile URL
+    flickr_user = flask.request.args.get("flickr_user", "")
+    flickr_user_url = f"https://www.flickr.com/photos/{flickr_username}/"
+
+    # Check for previous messages to this user
+    previous_messages = get_previous_messages(flickr_user, flickr_username)
+
+    # Get license code if provided
+    license_code = flask.request.args.get("license", type=int)
+    license_name = (
+        FLICKR_LICENSES.get(license_code, "") if license_code is not None else ""
+    )
+
+    is_free_license = license_code in FREE_LICENSES
+    is_nonfree_cc = license_code in NONFREE_CC_LICENSES
+
+    # For free licenses, show upload options instead of message
+    if is_free_license:
+        return flask.render_template(
+            "combined.html",
+            name=name,
+            enwp=enwp,
+            flickr_url=flickr_url,
+            img_url=img_url,
+            license_code=license_code,
+            license_name=license_name,
+            is_free_license=True,
+            wikipedia_url=wikipedia_url,
+            flickr_user=flickr_user,
+            flickr_user_url=flickr_user_url,
+            cat=cat,
+            previous_messages=previous_messages,
+        )
+
     msg = flask.render_template(
         "message.jinja",
         flickr_url=flickr_url,
@@ -652,6 +861,8 @@ def start() -> str:
         name=name,
         wiki_part1=wiki_part1,
         wiki_part2=wiki_part2,
+        is_nonfree_cc=is_nonfree_cc,
+        license_name=license_name,
     )
 
     subject = f"Request to use your photo of {name} on Wikipedia"
@@ -667,6 +878,43 @@ def start() -> str:
         lines=lines,
         nsid=nsid,
         img_url=img_url,
+        license_code=license_code,
+        license_name=license_name,
+        flickr_user=flickr_user,
+        flickr_user_url=flickr_user_url,
+        cat=cat,
+        previous_messages=previous_messages,
+    )
+
+
+@app.route("/category")
+def category_search() -> str:
+    """Find articles in a Wikipedia category that need images."""
+    cat = flask.request.args.get("cat", "").strip()
+
+    if not cat:
+        return flask.render_template("category.html")
+
+    category = parse_category_input(cat)
+    if not category:
+        return flask.render_template(
+            "category.html",
+            error="Invalid category format. Please enter a category name or URL.",
+            cat=cat,
+        )
+
+    articles, continue_token = get_articles_without_images(category)
+
+    # Get the display name (without Category: prefix)
+    category_name = category.replace("Category:", "")
+
+    return flask.render_template(
+        "category.html",
+        cat=cat,
+        category=category,
+        category_name=category_name,
+        articles=articles,
+        continue_token=continue_token,
     )
 
 

templates/category.html

@@ -0,0 +1,67 @@
+{% extends "base.html" %}
+
+{% block title %}Category Search - Flickr mail{% endblock %}
+
+{% block style %}
+<style>
+  .article-link:visited { color: #6f42c1; }
+</style>
+{% endblock %}
+
+{% block content %}
+  <div class="container">
+    <div class="row">
+    <h1>Find articles needing images</h1>
+    <p class="text-muted">Enter a Wikipedia category to find articles without images</p>
+
+    <form action="{{ url_for('category_search') }}" method="get">
+      <div class="mb-3">
+        <label for="cat" class="form-label">Wikipedia category name or URL:</label>
+        <input type="text" class="form-control" id="cat" name="cat" value="{{ cat }}"
+               placeholder="e.g., Living people or https://en.wikipedia.org/wiki/Category:Living_people" required>
+      </div>
+      <input type="submit" class="btn btn-primary" value="Search">
+      <a href="{{ url_for('start') }}" class="btn btn-outline-secondary ms-2">Back to main</a>
+    </form>
+
+    {% if error %}
+    <div class="alert alert-danger mt-3">{{ error }}</div>
+    {% endif %}
+
+    {% if category and articles is defined %}
+    <div class="mt-4">
+      <h5>Articles without images in <a href="https://en.wikipedia.org/wiki/{{ category | replace(' ', '_') }}" target="_blank">{{ category_name }}</a></h5>
+
+      {% if articles %}
+      <p class="text-muted small">Found {{ articles | length }} article(s) without images{% if continue_token %} (more available){% endif %}</p>
+
+      <div class="list-group">
+        {% for article in articles %}
+        <div class="list-group-item d-flex justify-content-between align-items-center">
+          <a href="{{ url_for('start', enwp=article.title, cat=cat) }}" class="text-decoration-none article-link">{{ article.title }}</a>
+          <a href="{{ article.wikipedia_url }}" target="_blank" class="badge bg-secondary text-decoration-none">Wikipedia</a>
+        </div>
+        {% endfor %}
+      </div>
+
+      {% if continue_token %}
+      <p class="text-muted small mt-3">Note: Only showing first batch of results. More articles may be available in this category.</p>
+      {% endif %}
+
+      {% else %}
+      <div class="alert alert-success mt-3">
+        All articles in this category have images!
+      </div>
+      {% endif %}
+    </div>
+    {% endif %}
+
+    <div class="mt-4">
+      <p class="text-muted small">
+        <a href="{{ url_for('start') }}">Back to Flickr mail home</a>
+      </p>
+    </div>
+
+    </div>
+  </div>
+{% endblock %}

templates/combined.html

@@ -13,6 +13,7 @@
       </div>
 
       <input type="submit" value="Submit">
+      <a href="{{ url_for('category_search') }}" class="btn btn-outline-secondary ms-2">Find articles by category</a>
     </form>
 
     {% if recent_uploads is defined and recent_uploads and not name %}
@@ -58,6 +59,9 @@
 
     {% if name and search_result is defined and search_result.photos %}
 
+    {% if cat %}
+    <p><a href="{{ url_for('category_search', cat=cat) }}">&larr; Back to category</a></p>
+    {% endif %}
     <p>Wikipedia article: {{ name }}</p>
     <p>Select a photo to compose a message ({{ search_result.total_photos | default(0) }} results):</p>
 
@@ -65,12 +69,12 @@
       {% for photo in search_result.photos %}
       <div class="col">
         <div class="card h-100">
-          <a href="{{ url_for(request.endpoint, enwp=enwp, flickr=photo.flickr_url, img=photo.medium_url) }}">
+          <a href="{{ url_for(request.endpoint, enwp=enwp, flickr=photo.flickr_url, img=photo.medium_url, license=photo.license, flickr_user=photo.realname or photo.username, cat=cat) }}">
             <img src="{{ photo.thumb_url }}" alt="{{ photo.title }}" class="card-img-top" style="aspect-ratio: 1; object-fit: cover;">
           </a>
           <div class="card-body p-2">
             <p class="card-text small mb-1 text-truncate" title="{{ photo.realname or photo.username }}">{{ photo.realname or photo.username }}</p>
-            <span class="badge {{ 'bg-success' if photo.license in [4, 5, 7, 8, 9, 10] else 'bg-secondary' }}">{{ photo.license_name }}</span>
+            <span class="badge {{ 'bg-success' if photo.license in [4, 5, 7, 8, 9, 10, 14, 15] else 'bg-secondary' }}">{{ photo.license_name }}</span>
           </div>
         </div>
       </div>
@@ -82,7 +86,7 @@
       <ul class="pagination justify-content-center">
         {% if search_result.current_page > 1 %}
         <li class="page-item">
-          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.current_page - 1) }}">Previous</a>
+          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.current_page - 1, cat=cat) }}">Previous</a>
         </li>
         {% else %}
         <li class="page-item disabled">
@@ -95,7 +99,7 @@
 
         {% if start_page > 1 %}
         <li class="page-item">
-          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=1) }}">1</a>
+          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=1, cat=cat) }}">1</a>
         </li>
         {% if start_page > 2 %}
         <li class="page-item disabled"><span class="page-link">...</span></li>
@@ -104,7 +108,7 @@
 
         {% for p in range(start_page, end_page + 1) %}
         <li class="page-item {{ 'active' if p == search_result.current_page else '' }}">
-          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=p) }}">{{ p }}</a>
+          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=p, cat=cat) }}">{{ p }}</a>
         </li>
         {% endfor %}
 
@@ -113,13 +117,13 @@
         <li class="page-item disabled"><span class="page-link">...</span></li>
         {% endif %}
         <li class="page-item">
-          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.total_pages) }}">{{ search_result.total_pages }}</a>
+          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.total_pages, cat=cat) }}">{{ search_result.total_pages }}</a>
         </li>
         {% endif %}
 
         {% if search_result.current_page < search_result.total_pages %}
         <li class="page-item">
-          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.current_page + 1) }}">Next</a>
+          <a class="page-link" href="{{ url_for(request.endpoint, enwp=enwp, page=search_result.current_page + 1, cat=cat) }}">Next</a>
         </li>
         {% else %}
         <li class="page-item disabled">
@@ -136,8 +140,11 @@
 
     {% elif name and not flickr_url %}
 
+    {% if cat %}
+    <p><a href="{{ url_for('category_search', cat=cat) }}">&larr; Back to category</a></p>
+    {% endif %}
     <p>Wikipedia article: {{ name }}</p>
-    <p class="text-warning">No photos found. Try a different search term.</p>
+    <div class="alert alert-warning">No photos found. Try a different search term.</div>
     <p><a href="https://flickr.com/search/?view_all=1&text={{ '"' + name + '"' | urlencode }}" target="_blank">Search on Flickr directly</a></p>
 
     {% endif %}
@@ -155,7 +162,36 @@
       {% else %}
       <div class="col-12">
       {% endif %}
+        {% if flickr_user %}
+        <p><strong>User:</strong> <a href="{{ flickr_user_url }}" target="_blank">{{ flickr_user }}</a></p>
+        {% endif %}
+        {% if previous_messages %}
+        <div class="alert alert-info">
+          <strong>Previously contacted:</strong> You have sent {{ previous_messages | length }} message(s) to this user.
+          <ul class="mb-0 mt-2">
+            {% for msg in previous_messages %}
+            <li><a href="{{ msg.url }}" target="_blank">{{ msg.subject }}</a> <small class="text-muted">({{ msg.date }})</small></li>
+            {% endfor %}
+          </ul>
+        </div>
+        {% endif %}
+        {% if is_free_license %}
+        <div class="alert alert-success">
+          <strong>Ready to upload!</strong> This photo is licensed under <span class="badge bg-success">{{ license_name }}</span> and can be used on Wikipedia.
+        </div>
+        <p>
+          <a href="https://commons.wikimedia.org/wiki/Special:UploadWizard" class="btn btn-primary" target="_blank">
+            Upload to Wikimedia Commons
+          </a>
+        </p>
+        <p class="text-muted small">
+          After uploading, you can add the image to the Wikipedia article for <a href="{{ wikipedia_url }}" target="_blank">{{ name }}</a>.
+        </p>
+        {% else %}
         <p><a href="https://www.flickr.com/mail/write/?to={{nsid}}" class="btn btn-primary">Send message on Flickr</a></p>
+        {% if license_name %}
+        <div class="mb-2"><strong>Current license:</strong> <span class="badge {{ 'bg-success' if license_code in [4, 5, 7, 8, 9, 10, 14, 15] else 'bg-warning text-dark' if license_code in [1, 2, 3, 6, 11, 12, 13, 16] else 'bg-secondary' }}">{{ license_name }}</span></div>
+        {% endif %}
         <div class="mb-2"><strong>Subject:</strong> {{ subject }} <button class="btn btn-sm btn-outline-secondary" id="copy-subject">copy</button></div>
         <div>
           <h5>Message <button class="btn btn-sm btn-outline-secondary" id="copy-message">copy</button></h5>
@@ -163,6 +199,10 @@
           <p>{{ p }}</p>
           {% endfor %}
         </div>
+        {% endif %}
+        <p class="mt-3">
+          <a href="{{ url_for('start', enwp=enwp, cat=cat) if cat else url_for('start', enwp=enwp) }}">&larr; Back to search results</a>
+        </p>
       </div>
     </div>
     {% endif %}

templates/message.jinja

@@ -1,5 +1,33 @@
 {# vim:ft=jinja
 #}
+{% if is_nonfree_cc %}
+Hi,
+
+I'd like to use your photo to illustrate the article about {{ name }} on Wikipedia.
+
+{{ flickr_url }}
+
+{{ wiki_part1 }}{{ wiki_part2 | urlencode }}
+
+I noticed your photo is licensed under {{ license_name }}. Thank you for sharing your work with a Creative Commons license! Unfortunately, Wikipedia can only use images that allow commercial use and derivative works, so the current license restrictions prevent us from using it.
+
+{% if 'NC' in license_name and 'ND' in license_name %}
+The "NonCommercial" restriction means the image can't be used on Wikipedia because Wikipedia content may be reused commercially. The "NoDerivs" restriction also prevents use because Wikipedia needs to allow image cropping and other modifications.
+{% elif 'NC' in license_name %}
+The "NonCommercial" restriction means the image can't be used on Wikipedia because Wikipedia content may be reused commercially by anyone.
+{% elif 'ND' in license_name %}
+The "NoDerivs" restriction means the image can't be used on Wikipedia because Wikipedia needs to allow modifications like cropping or colour correction.
+{% endif %}
+
+Would you consider changing the license to Creative Commons Attribution (CC BY) or Attribution-ShareAlike (CC BY-SA)? These licenses still require credit to be given to you, but allow the broader use that Wikipedia requires.
+
+To adjust the license settings, you can click on the license name on the right-hand side of the photo's page, just underneath the date.
+
+Thanks,
+
+Edward
+edward@4angle.com
+{% else %}
 Hi,
 
 I'd like to use your photo to illustrate the article about {{ name }} on Wikipedia.
@@ -18,3 +46,4 @@ Thanks,
 
 Edward
 edward@4angle.com
+{% endif %}