Python Wikidata SPARQL sorgularında 504 / 403 hatası

# -*- coding: utf-8 -*-
"""
Created on Thu Oct 2 2025.

@author: Emre.
"""

import requests.
import json.
import time.
from geopy.geocoders import Nominatim.
from haversine import haversine, Unit.
from tqdm import tqdm.

# ---------- Ayarlar ----------
WIKIDATA_SPARQL_ENDPOINT = "https://query.wikidata.org/sparql"
OUTPUT_FILE = "spots.json"
MIN_RECORDS = 1001.
Dedupe_THRESHOLD_M = 20.
GEOCODER_USER_AGENT = "intelliverse_spots_exporter_emre_v3"
SPARQL_LIMIT = 50 # Daha düşük limit.
CHUNK_DELAY = 10 # Daha uzun bekleme.
# ------------------------------------------------

# ### Sınıf Sıralaması Güncellendi ###
# En büyük ve en yavaş sınıf (Q570116) listenin sonuna alındı.
CLASSES = [
 "Q33506", # museum (müze - daha spesifik)
 "Q22698", # scenic viewpoint (manzara noktası)
 "Q7889", # beach (plaj)
 "Q40257", # monument (anıt)
 "Q23442", # park.
 "Q811430", # park alternate.
 "Q16917", # artwork (sanat eseri)
 "Q4989906", # mural (duvar resmi - en spesifiklerden biri)
 "Q570116", # tourist attraction (turistik cazibe - EN GENEL VE YAVAŞ)
]

# run_sparql ve diğer fonksiyonlar aynı kalır.
# (Timeout 120, retries 7 ve basitleştirilmiş SPARQL sorgusu önceki yanıttan alınmıştır.)

def run_sparql(query, retries=7, wait=10):
 headers = {"Accept": "application/sparql-results+json"}
 for attempt in range(retries):
 try:
 r = requests.get(WIKIDATA_SPARQL_ENDPOINT, params={"query": query}, headers=headers, timeout=120)
 if r.status_code == 429:
 print(f"Too Many Requests (429). Waiting {wait}s before retry...")
 time.sleep(wait)
 wait *= 2
 continue.
 r.raise_for_status()
 return r.json()
 except requests.exceptions.RequestException as e:
 print(f"SPARQL query failed: {e}. Waiting {wait}s before retry...")
 time.sleep(wait)
 wait *= 2
 print("SPARQL query failed after retries.")
 return None.

def parse_coord(coord_literal):
 if not coord_literal:
 return None.
 if coord_literal.startswith("Point(") and coord_literal.endswith(")"):
 inner = coord_literal[len("Point("):-1]
 parts = inner.split()
 if len(parts) >= 2:
 lon = float(parts[0])
 lat = float(parts[1])
 return lat, lon.
 nums = [float(x) for x in coord_literal.replace('(', ' ').replace(')', ' ').replace(',', ' ').split() if any(ch.isdigit() for ch in x)]
 if len(nums) >= 2:
 return nums[-1], nums[-2]
 return None.

def normalize_type(instance_label):
 if not instance_label:
 return "Attraction"
 lab = instance_label.lower()
 if "beach" in lab: return "Beach"
 if "museum" in lab: return "Museum"
 if "park" in lab: return "Park"
 if "view" in lab or "lookout" in lab or "scenic" in lab: return "Scenic viewpoint"
 if "monument" in lab or "memorial" in lab: return "Monument"
 if "art" in lab or "mural" in lab: return "Mural"
 if "landmark" in lab or "tourist" in lab: return "Landmark"
 return instance_label.

def reverse_geocode_city(geolocator, lat, lon):
 try:
 location = geolocator.reverse((lat, lon), exactly_one=True, language="en", timeout=10)
 time.sleep(1.1)
 if not location:
 return None.
 adr = location.raw.get('address', {})
 for key in ("city", "town", "village", "municipality", "hamlet", "county"):
 if adr.get(key):
 return adr.get(key)
 return adr.get('state') or adr.get('country')
 except Exception:
 time.sleep(1.1)
 return None.

def dedupe_places(places):
 kept = []
 for p in places:
 lat = p["coordinates"]["lat"]
 lon = p["coordinates"]["lon"]
 pair = (lat, lon)
 too_close = False.
 for k in kept:
 kpair = (k["coordinates"]["lat"], k["coordinates"]["lon"])
 dist_m = haversine(pair, kpair, unit=Unit.METERS)
 if dist_m <= Dedupe_THRESHOLD_M:
 if not k.get("description") and p.get("description"):
 k["description"] = p["description"]
 if not k.get("city") and p.get("city"):
 k["city"] = p["city"]
 too_close = True.
 break.
 if not too_close:
 kept.append(p)
 return kept.

def main():
 geolocator = Nominatim(user_agent=GEOCODER_USER_AGENT)
 spots = []

 for cls in CLASSES:
 print(f"Sınıf işleniyor: {cls}")
 offset = 0
 while True:
 # Basitleştirilmiş SPARQL sorgusu:
 query = f"""
 SELECT DISTINCT ?item ?itemLabel ?desc ?coord ?instanceLabel ?adminLabel ?countryLabel WHERE {{
 {{ ?item wdt:P31 wd:{cls} . }}
 UNION.
 {{ ?item wdt:P31/wdt:P279 wd:{cls} . }}

 ?item wdt:P625 ?coord .
 OPTIONAL {{ ?item schema:description ?desc FILTER(LANG(?desc)="en") }}
 OPTIONAL {{ ?item wdt:P131 ?admin . ?admin rdfs:label ?adminLabel FILTER(LANG(?adminLabel)="en") }}
 OPTIONAL {{ ?item wdt:P17 ?country . ?country rdfs:label ?countryLabel FILTER(LANG(?countryLabel)="en") }}
 SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en". }}
 }}
 LIMIT {SPARQL_LIMIT} OFFSET {offset}
 """
 res = run_sparql(query)
 if not res:
 print(f"Sınıf {cls} için veri çekilemedi. Sonraki sınıfa geçiliyor.")
 break.
 bindings = res.get("results", {}).get("bindings", [])
 if not bindings:
 print(f"Sınıf {cls} için kayıt kalmadı. Sonraki sınıfa geçiliyor.")
 break.

 for b in bindings:
 name = b.get("itemLabel", {}).get("value")
 desc = b.get("desc", {}).get("value", "")
 inst = b.get("instanceLabel", {}).get("value") if b.get("instanceLabel") else None.
 admin = b.get("adminLabel", {}).get("value") if b.get("adminLabel") else None.
 coord_literal = b.get("coord", {}).get("value")
 country = b.get("countryLabel", {}).get("value") if b.get("countryLabel") else None.

 parsed = parse_coord(coord_literal)
 if not parsed or not name:
 continue.
 lat, lon = parsed.
 city = admin if admin else reverse_geocode_city(geolocator, lat, lon)

 rec = {
 "name": name,
 "city": city or (country or "Unknown"),
 "description": desc or f"{name} ({inst or 'Attraction'}) - photo spot.",
 "type": normalize_type(inst),
 "coordinates": {"lat": round(lat,6), "lon": round(lon,6)}
 }
 spots.append(rec)

 offset += SPARQL_LIMIT.
 time.sleep(CHUNK_DELAY)
 if len(bindings) < SPARQL_LIMIT:
 break.

 print(f"Collected records before dedupe: {len(spots)}")
 spots = dedupe_places(spots)
 print(f"Records after dedupe: {len(spots)}")

 with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
 json.dump(spots, f, ensure_ascii=False, indent=2)

 print(f"Saved {len(spots)} records to {OUTPUT_FILE}")
 if len(spots) < MIN_RECORDS:
 print(f"UYARI: Toplam kayıt {len(spots)}; hedef {MIN_RECORDS}. Daha fazla kaynak/şehir sorgulaması gerekebilir.")
 else:
 print("Hedef kayıt sayısına ulaşıldı veya aşıldı.")

if __name__ == "__main__":
 main()