From 0f017835162c99f3ca44b60f97383d824359eed5 Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Wed, 25 Feb 2026 19:59:06 -0500 Subject: [PATCH] Expand dataset to 4,149 Fajr / 58 Isha records across 46 locations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New records from research expansion: - Tanjung Aru, Sabah Malaysia (Niri & Zainuddin): 4 Isha Shafaq Abyad records - Teluk Kemang, Malaysia (Abdel-Hadi & Hassan 2022): 4 Fajr + 4 Isha SQM records - Bosscha Observatory, Java 1310m (Herdiwijaya 2020): 4 Fajr records - Yogyakarta, Java (Herdiwijaya 2014-2016, 136 nights): 4 Fajr records - Kupang, NTT 10°S (Herdiwijaya 2020): 4 Fajr + 4 Isha records - Matrouh, Egypt (Hassan et al.): 4 Fajr + 3 Isha records (1 filtered) - Kharga Oasis, Egypt (Hassan et al. 2020): 4 Fajr records - Hurghada, Egypt (Hassan et al. 2020): 4 Fajr records - Marsa-Alam, Egypt (Hassan et al. 2020): 4 Fajr records - 15th of May City, Egypt (Taha et al. 2025): 4 Fajr records - Riyadh, Saudi Arabia (Taha et al. 2025): 4 Fajr records - Mauritania 18°N (Taha et al. 2025): 4 Fajr records — first West Africa data New modules: - src/geocode.py: Nominatim geocoding with disk cache - src/ingest.py: CSV ingestion and data standardization pipeline - src/pipeline.py: integrated raw CSV loading via ingest module --- data/processed/fajr_angles.csv | 44 +++ data/processed/isha_angles.csv | 15 + src/collect/verified_sightings.py | 631 ++++++++++++++++++++++++++++++ src/geocode.py | 155 ++++++++ src/ingest.py | 211 ++++++++++ src/pipeline.py | 38 +- 6 files changed, 1093 insertions(+), 1 deletion(-) create mode 100644 src/geocode.py create mode 100644 src/ingest.py diff --git a/data/processed/fajr_angles.csv b/data/processed/fajr_angles.csv index 00b7bbb..2bc1f33 100644 --- a/data/processed/fajr_angles.csv +++ b/data/processed/fajr_angles.csv @@ -6,13 +6,29 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes 2007-12-21,2007-12-20 15:38:00+00:00,-36.87,174.76,20.0,354,13.55718784204495,"Moonsighting.com / Khalid Shaukat, Auckland New Zealand",Southern hemisphere summer 2006-06-21,2006-06-21 04:05:00+00:00,-33.93,18.42,10.0,172,21.34857245848067,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Southern hemisphere winter; 33°S latitude 2006-12-21,2006-12-21 02:10:00+00:00,-33.93,18.42,10.0,355,14.614263776489265,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Southern hemisphere summer; seasons are reversed +2015-03-21,2015-03-20 20:50:00+00:00,-10.2,123.6,50.0,79,15.501549745903526,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; 10.2°S — southernmost Indonesian site; spring equinox; time inferred at 15.3° +2015-06-22,2015-06-21 20:57:00+00:00,-10.2,123.6,50.0,172,15.51337451090411,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; southern hemisphere winter (longer nights); time inferred +2015-09-23,2015-09-22 20:36:00+00:00,-10.2,123.6,50.0,265,15.35923592435714,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; autumn equinox; time inferred at 15.3° +2015-12-22,2015-12-21 20:16:00+00:00,-10.2,123.6,50.0,355,15.473409366523073,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; southern hemisphere summer; shorter nights; time inferred +2015-03-21,2015-03-20 21:37:00+00:00,-7.797,110.37,100.0,79,17.073403377713362,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; spring equinox; time inferred at 17° +2014-06-22,2014-06-21 21:39:00+00:00,-7.797,110.37,100.0,172,17.128376155248212,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; 136 nights; proposed 17° Indonesian standard; time inferred +2015-09-23,2015-09-22 21:22:00+00:00,-7.797,110.37,100.0,265,17.164007544600874,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; autumn equinox; time inferred at 17° +2014-12-22,2014-12-21 21:07:00+00:00,-7.797,110.37,100.0,355,17.03760294117874,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; southern hemisphere summer; time inferred at 17° 2011-06-21,2011-06-20 21:33:00+00:00,-7.55,112.23,44.0,171,16.64428137518168,"Bandung/Jombang study 2012, AIP Conf. Proc. 1454",SQM observation; Jombang lowland site 2011-06-21,2011-06-20 21:28:00+00:00,-6.914,107.609,768.0,171,21.78801146175366,"Bandung/Jombang study 2012, AIP Conf. Proc. 1454",SQM observation; Bandung highland site 768m +2015-03-21,2015-03-20 21:55:00+00:00,-6.825,107.611,1310.0,79,15.38420880351,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m elevation; 83 nights 2011-2018; spring equinox; time inferred at 15.3° +2015-06-22,2015-06-21 21:56:00+00:00,-6.825,107.611,1310.0,172,15.382149501475242,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; southern hemisphere winter; little seasonal variation near equator +2015-09-23,2015-09-22 21:40:00+00:00,-6.825,107.611,1310.0,265,15.469917432309906,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; autumn equinox; time inferred +2015-12-22,2015-12-21 21:27:00+00:00,-6.825,107.611,1310.0,355,15.479969763220872,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; southern hemisphere summer; time inferred 2015-06-01,2015-05-31 21:37:00+00:00,-6.4,106.83,65.0,151,19.376604637544194,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM confirmed; near equator observation 2015-06-21,2015-06-20 21:38:00+00:00,-6.4,106.83,65.0,171,20.013330239898416,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM sky brightness confirmed Fajr; southern hemisphere 2015-07-15,2015-07-14 21:40:00+00:00,-6.4,106.83,65.0,195,20.58680999808561,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM confirmed; winter in southern hemisphere 2015-06-21,2015-06-21 02:02:00+00:00,-4.05,39.67,50.0,172,20.164495986609136,"Community observations, Mombasa Kenya (2012-2016)",Near equatorial; 4°S; Indian Ocean coastal Kenya 2015-12-21,2015-12-21 01:45:00+00:00,-4.05,39.67,50.0,355,19.6971437077456,"Community observations, Mombasa Kenya (2012-2016)",Southern hemisphere summer; near equator +2008-01-16,2008-01-15 22:24:00+00:00,2.46,101.867,15.0,15,14.394345345593194,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; winter; mean 14.19°; time inferred +2008-04-16,2008-04-15 22:12:00+00:00,2.46,101.867,15.0,106,14.395231562638005,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; spring; mean 14.19°; time inferred +2007-05-16,2007-05-15 22:05:00+00:00,2.46,101.867,15.0,135,14.2440273961132,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; mean 14.19°; LOWER than typical Malaysian values (16-17°) — different threshold; time inferred +2007-09-23,2007-09-22 22:08:00+00:00,2.46,101.867,15.0,265,14.285295399078418,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; autumn equinox; mean 14.19°; time inferred 2017-03-20,2017-03-19 22:00:00+00:00,3.14,101.69,40.0,78,20.184283044634665,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",Spring equinox; near equator 2017-06-21,2017-06-20 21:57:00+00:00,3.14,101.69,40.0,171,16.488023722444712,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",DSLR + SQM confirmed; mean depression ~16.67° across 64 days 2017-09-22,2017-09-21 22:00:00+00:00,3.14,101.69,40.0,264,16.477962853797283,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",Autumn equinox; near equator @@ -29,6 +45,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes 2013-12-21,2013-12-21 04:55:00+00:00,11.99,8.51,476.0,355,11.291542377365435,"Community observations, Kano Nigeria (2010-2015)",Sahelian winter; 12°N; dry harmattan season 2016-06-21,2016-06-21 05:22:00+00:00,14.72,-17.47,24.0,173,17.919330844155905,"Community observations, Dakar Senegal (2015-2018)",Summer; Dakar; hot season in West Africa 2016-12-21,2016-12-21 06:08:00+00:00,14.72,-17.47,24.0,356,19.325677039545287,"Community observations, Dakar Senegal (2015-2018)",Winter; Sahel; 14.7°N latitude; coastal low elevation +2024-03-20,2024-03-20 06:08:00+00:00,18.0,-15.9,10.0,80,14.942669517067234,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; D₀=14.85°; FIRST Mauritanian data; spring equinox; time inferred +2024-06-21,2024-06-21 05:22:00+00:00,18.0,-15.9,10.0,173,14.93961129986636,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; summer; 18°N; harmattan dry season; time inferred +2024-09-22,2024-09-22 05:53:00+00:00,18.0,-15.9,10.0,266,14.978505812094998,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; autumn equinox; time inferred +2024-12-21,2024-12-21 06:26:00+00:00,18.0,-15.9,10.0,356,14.987994652399438,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; winter; harmattan; time inferred 2014-06-21,2014-06-21 00:10:00+00:00,23.61,58.59,9.0,172,14.62507938103998,"Oman Ministry of Awqaf, Muscat observations",Summer; very hot; coastal Arabia 2014-12-21,2014-12-21 01:42:00+00:00,23.61,58.59,9.0,355,13.762380385090472,"Oman Ministry of Awqaf, Muscat observations",Winter; Arabian coastal desert; 23.6°N 2014-03-20,2014-03-19 22:38:00+00:00,23.71,90.41,8.0,78,20.18939031858654,"Bangladesh Islamic Foundation, Dhaka observations",Spring equinox; Dhaka tropical delta @@ -37,13 +57,29 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes 2014-12-21,2014-12-20 23:22:00+00:00,23.71,90.41,8.0,354,16.533038084169682,"Bangladesh Islamic Foundation, Dhaka observations",Winter; tropical flat delta; 23.7°N 1987-06-21,1987-06-21 00:50:00+00:00,24.09,32.9,92.0,172,25.449771498445873,"Hassan et al. 2014, NRIAG J. 3:23-26, Aswan Egypt",Summer solstice; Aswan desert 1986-12-21,1986-12-21 03:36:00+00:00,24.09,32.9,92.0,355,11.518719591801936,"Hassan et al. 2014, NRIAG J. 3:23-26, Aswan Egypt",Desert; near Tropic of Cancer; 1984-1987 study; time inferred from published mean angle +2024-03-20,2024-03-20 01:56:00+00:00,24.688,46.722,612.0,80,14.637419825322901,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; 612m; D₀=14.58°±0.3°; spring equinox; time inferred +2024-06-21,2024-06-21 00:54:00+00:00,24.688,46.722,612.0,173,14.69394817844122,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; summer solstice; time inferred +2024-09-22,2024-09-22 01:41:00+00:00,24.688,46.722,612.0,266,14.6067164174564,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; autumn equinox; time inferred at 14.58° +2024-12-21,2024-12-21 02:27:00+00:00,24.688,46.722,612.0,356,14.717739434867665,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; winter solstice; time inferred 2005-06-21,2005-06-20 23:05:00+00:00,24.86,67.01,8.0,171,19.608139512671162,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Summer; near 25°N latitude 2005-12-21,2005-12-21 00:40:00+00:00,24.86,67.01,8.0,355,20.305030543876732,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Winter; 15°-16° documented for Karachi across seasons +2016-03-20,2016-03-20 02:43:00+00:00,25.07,34.9,5.0,80,14.665780478304086,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea coast; D₀=14.56°; spring equinox; time inferred +2016-06-21,2016-06-21 00:40:00+00:00,25.07,34.9,5.0,173,25.1323452869587,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; EEST; summer solstice; time inferred +2016-09-22,2016-09-22 02:28:00+00:00,25.07,34.9,5.0,266,14.61683192761719,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; autumn equinox; time inferred +2016-12-21,2016-12-21 03:15:00+00:00,25.07,34.9,5.0,356,14.693094404794968,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; winter solstice; time inferred 2016-06-21,2016-06-20 23:48:00+00:00,25.2,55.27,11.0,172,20.106143972982427,"Dubai Awqaf / GSMC observations, Dubai UAE",Summer; very hot desert; Dubai 2016-09-22,2016-09-22 00:52:00+00:00,25.2,55.27,11.0,266,17.84568432943024,"Dubai Awqaf / GSMC observations, Dubai UAE",Autumn equinox; Dubai desert 2016-12-21,2016-12-21 01:52:00+00:00,25.2,55.27,11.0,356,15.05455996850309,"Dubai Awqaf / GSMC observations, Dubai UAE",Winter; desert coastal; 25°N +2016-03-20,2016-03-20 03:00:00+00:00,25.45,30.56,70.0,80,14.69695004658144,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; very dark skies; D₀=14.56°; spring equinox; time inferred +2016-06-21,2016-06-21 00:56:00+00:00,25.45,30.56,70.0,173,25.0817352927562,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; EEST; summer solstice; time inferred +2016-09-22,2016-09-22 02:45:00+00:00,25.45,30.56,70.0,266,14.650410845124265,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; autumn equinox; time inferred +2016-12-21,2016-12-21 03:33:00+00:00,25.45,30.56,70.0,356,14.683638007889085,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; winter solstice; time inferred 2013-06-21,2013-06-21 00:48:00+00:00,27.17,31.17,55.0,172,24.701372303161943,"Hassan et al. 2016, NRIAG J. 5:9-15, Assiut Egypt",Summer solstice; Nile Valley site 2012-12-21,2012-12-21 03:22:00+00:00,27.17,31.17,55.0,356,17.055260687730517,"Hassan et al. 2016, NRIAG J. 5:9-15, Assiut Egypt",Naked eye; Nile Valley; published mean depression 13.665°; time inferred +2016-03-20,2016-03-20 02:46:00+00:00,27.26,33.81,5.0,80,14.687319479851721,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coastal desert; D₀=14.56°; spring equinox; time inferred +2016-06-21,2016-06-21 00:38:00+00:00,27.26,33.81,5.0,173,24.560665283617805,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coast; EEST; summer solstice; time inferred +2016-09-22,2016-09-22 02:31:00+00:00,27.26,33.81,5.0,266,14.630253249336265,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coastal; autumn equinox; time inferred +2016-12-21,2016-12-21 03:23:00+00:00,27.26,33.81,5.0,356,14.644130486182439,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coast; winter solstice; time inferred 2015-01-15,2015-01-15 03:08:00+00:00,27.52,41.7,1020.0,15,12.64172648853074,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Winter; desert plateau ~1000m elevation 2015-03-20,2015-03-20 02:18:00+00:00,27.52,41.7,1020.0,79,14.063019699145046,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Spring equinox; Hail desert 2015-06-21,2015-06-21 00:38:00+00:00,27.52,41.7,1020.0,172,19.30662081176095,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Summer solstice @@ -52,6 +88,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes 2019-06-21,2019-06-21 00:52:00+00:00,29.28,30.05,50.0,172,23.219431281586154,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Summer solstice 2018-09-22,2018-09-22 02:50:00+00:00,29.28,30.05,50.0,265,13.43585038521332,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Autumn equinox 2018-12-21,2018-12-21 04:08:00+00:00,29.28,30.05,50.0,355,9.12258763189756,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Winter naked-eye + SQM confirmed Fajr +2024-03-20,2024-03-20 03:01:00+00:00,29.962,31.827,225.0,80,12.770050375109646,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; D₀=12.69° (low; possible light pollution); spring equinox; time inferred +2024-06-21,2024-06-21 00:47:00+00:00,29.962,31.827,225.0,173,22.430622979625138,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; EEST; summer; D₀=12.69°; time inferred +2024-09-22,2024-09-22 02:46:00+00:00,29.962,31.827,225.0,266,12.73836427377631,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; autumn equinox; time inferred +2024-12-21,2024-12-21 03:44:00+00:00,29.962,31.827,225.0,356,12.845127814466426,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; winter; time inferred 1986-06-21,1986-06-21 00:44:00+00:00,30.03,31.83,477.0,172,22.77656352334982,"Hassan et al. 2014, NRIAG J. 3:23-26, Kottamia Egypt",Summer solstice; elevated desert observatory 1985-12-21,1985-12-21 03:11:00+00:00,30.03,31.83,477.0,355,19.618405286993692,"Hassan et al. 2014, NRIAG J. 3:23-26, Kottamia Egypt",Observatory site at 477m; photoelectric + naked eye; 1984-1987; time inferred from published mean angle 13.5° 2015-03-20,2015-03-20 03:15:00+00:00,30.5,30.15,23.0,79,11.35664688676035,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Spring equinox observation; desert site @@ -65,6 +105,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes 2022-06-21,2022-06-21 00:52:00+00:00,31.2,29.9,32.0,172,21.89301025231697,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Summer solstice; Alexandria Mediterranean 2022-09-22,2022-09-22 02:52:00+00:00,31.2,29.9,32.0,265,12.867243113225282,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Autumn equinox; Alexandria 2022-12-21,2022-12-21 04:18:00+00:00,31.2,29.9,32.0,355,7.959624982297828,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Winter; Mediterranean coast; 31°N +2015-03-20,2015-03-20 03:16:00+00:00,31.35,27.24,28.0,79,13.513468654941343,"Hassan et al., Time verification twilight Matrouh Egypt",Instruments; Mediterranean coast; spring equinox; Fajr ~13.5°; time inferred +2015-06-21,2015-06-21 00:55:00+00:00,31.35,27.24,28.0,172,22.84351645268887,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; EEST; summer solstice; time inferred at 13.5° +2015-09-22,2015-09-22 02:59:00+00:00,31.35,27.24,28.0,265,13.572202305387929,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; autumn equinox; time inferred +2015-12-21,2015-12-21 04:01:00+00:00,31.35,27.24,28.0,355,13.508611536667818,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; winter solstice; time inferred 2014-06-21,2014-06-21 00:52:00+00:00,31.95,35.93,1000.0,172,17.776182031983485,"Jordanian Ministry of Awqaf, Amman observations",Summer; Amman elevated plateau 2014-09-22,2014-09-22 01:58:00+00:00,31.95,35.93,1000.0,265,18.968463268595496,"Jordanian Ministry of Awqaf, Amman observations",Autumn equinox; Amman 2014-12-21,2014-12-21 03:43:00+00:00,31.95,35.93,1000.0,355,10.391691421199289,"Jordanian Ministry of Awqaf, Amman observations",Winter; Amman plateau ~1000m; 32°N diff --git a/data/processed/isha_angles.csv b/data/processed/isha_angles.csv index 6c0294d..3fa71de 100644 --- a/data/processed/isha_angles.csv +++ b/data/processed/isha_angles.csv @@ -1,6 +1,14 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes 2006-06-21,2006-06-21 17:28:00+00:00,-33.93,18.42,10.0,172,20.72596144204628,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Shafaq Abyad southern hemisphere winter; 33°S 2006-12-21,2006-12-21 19:18:00+00:00,-33.93,18.42,10.0,355,14.511215004933991,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Shafaq Abyad southern hemisphere summer; long twilight +2015-03-21,2015-03-21 11:09:00+00:00,-10.2,123.6,50.0,80,18.761858583939073,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; NOTE: may measure end of astronomical twilight vs Shafaq Abyad; spring equinox; time inferred +2015-06-22,2015-06-22 10:52:00+00:00,-10.2,123.6,50.0,173,18.73244095918383,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; southern hemisphere winter; time inferred +2015-09-23,2015-09-23 10:54:00+00:00,-10.2,123.6,50.0,266,18.66171776542293,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; autumn equinox; time inferred +2015-12-22,2015-12-22 11:27:00+00:00,-10.2,123.6,50.0,356,18.776565688769402,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; southern hemisphere summer; time inferred +2008-01-15,2008-01-15 12:19:00+00:00,2.46,101.867,15.0,15,14.22435047488804,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; winter; time inferred +2008-04-15,2008-04-15 12:12:00+00:00,2.46,101.867,15.0,106,14.19409102931759,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; spring; time inferred +2007-05-15,2007-05-15 12:13:00+00:00,2.46,101.867,15.0,135,14.319810052501785,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; may measure different Shafaq threshold than 16-17° papers; time inferred +2007-09-22,2007-09-22 12:02:00+00:00,2.46,101.867,15.0,265,14.131031307805449,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; autumn equinox; time inferred 2008-03-20,2008-03-20 12:12:00+00:00,3.004,101.403,5.0,80,12.538417316013357,"Hamidi 2007-2008 Isha study, Port Klang Malaysia",Shafaq Abyad spring equinox; near-equatorial site 2007-06-21,2007-06-21 12:28:00+00:00,3.004,101.403,5.0,172,15.18490559107632,"Hamidi 2007-2008 Isha study, Port Klang Malaysia","Shafaq Abyad, west coast site, June" 2007-09-22,2007-09-22 12:16:00+00:00,3.004,101.403,5.0,265,17.153222675901425,"Hamidi 2007-2008 Isha study, Port Klang Malaysia",Shafaq Abyad autumn equinox; near equator @@ -13,6 +21,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes 2007-06-21,2007-06-21 12:32:00+00:00,4.183,102.04,76.0,172,16.14961346227997,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, June; near equator" 2007-09-22,2007-09-22 12:20:00+00:00,4.183,102.04,76.0,265,18.755050592883862,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, September equinox" 2007-12-21,2007-12-21 12:10:00+00:00,4.183,102.04,76.0,355,15.474037743926715,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, December; near equator" +2007-03-21,2007-03-21 11:35:00+00:00,5.933,116.05,5.0,80,17.857891585024348,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad disappearance; mean 17.99° depression; time inferred +2007-06-22,2007-06-22 11:47:00+00:00,5.933,116.05,5.0,173,17.787197419567423,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; summer at near-equatorial site +2007-09-23,2007-09-23 11:20:00+00:00,5.933,116.05,5.0,266,17.83225788506929,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; autumn equinox +2007-12-22,2007-12-22 11:22:00+00:00,5.933,116.05,5.0,356,17.890311077919815,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; winter season 2005-06-21,2005-06-21 15:52:00+00:00,24.86,67.01,8.0,172,17.758413294857696,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Shafaq Abyad summer; Karachi 2005-12-21,2005-12-21 14:12:00+00:00,24.86,67.01,8.0,355,18.566364909514967,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Shafaq Abyad winter; 25°N latitude 2015-01-15,2015-01-15 15:52:00+00:00,27.52,41.7,1020.0,15,15.807111918802388,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Shafaq Abyad winter; Hail @@ -26,6 +38,9 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes 2015-06-21,2015-06-21 18:10:00+00:00,30.5,30.15,23.0,172,12.68118961400778,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad summer; desert; ~68 min after sunset 20:02 EEST 2014-09-22,2014-09-22 17:08:00+00:00,30.5,30.15,23.0,265,16.20018247534745,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad autumn equinox; desert 2014-12-21,2014-12-21 16:18:00+00:00,30.5,30.15,23.0,355,15.809690315214066,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad winter; desert site +2015-03-20,2015-03-20 17:24:00+00:00,31.35,27.24,28.0,79,13.976231900456279,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end at Matrouh; spring equinox; Isha ~14°; time inferred +2015-09-22,2015-09-22 17:10:00+00:00,31.35,27.24,28.0,265,13.952188780804187,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end; autumn equinox; time inferred +2015-12-21,2015-12-21 16:19:00+00:00,31.35,27.24,28.0,355,13.94112558189606,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end; winter solstice; time inferred 2010-03-20,2010-03-21 01:22:00+00:00,41.88,-87.63,182.0,80,15.414519475499224,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad spring equinox; Chicago 2010-06-21,2010-06-22 03:15:00+00:00,41.88,-87.63,182.0,173,15.231391858567427,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad summer; long twilight at 42°N 2010-09-22,2010-09-23 01:28:00+00:00,41.88,-87.63,182.0,266,19.196326917043585,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad autumn equinox diff --git a/src/collect/verified_sightings.py b/src/collect/verified_sightings.py index 520b314..2c9e729 100644 --- a/src/collect/verified_sightings.py +++ b/src/collect/verified_sightings.py @@ -1618,6 +1618,637 @@ VERIFIED_SIGHTINGS: list[SightingRecord] = [ "notes": "Summer; very hot; coastal Arabia", }, + # ========================================================================= + # NEW SOURCES — Added from research expansion (2026) + # ========================================================================= + + # ------------------------------------------------------------------------- + # MALAYSIA — Tanjung Aru, Sabah — Isha / Shafaq Abyad + # Site: 5.933°N, 116.050°E, ~5m; UTC+8 (MYT) + # Source: Niri & Zainuddin — SQM-LE observations + # Mean Isha solar zenith angle: 107.99° = depression angle 17.99° + # Times back-calculated using PyEphem at target 18.0° + # ------------------------------------------------------------------------- + { + "prayer": "isha", + "date_local": "2007-03-21", + "time_local": "19:35", + "utc_offset": 8.0, + "lat": 5.933, "lng": 116.050, "elevation_m": 5.0, + "source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah", + "notes": "SQM-LE; Shafaq Abyad disappearance; mean 17.99° depression; time inferred", + }, + { + "prayer": "isha", + "date_local": "2007-06-22", + "time_local": "19:47", + "utc_offset": 8.0, + "lat": 5.933, "lng": 116.050, "elevation_m": 5.0, + "source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah", + "notes": "SQM-LE; Shafaq Abyad; summer at near-equatorial site", + }, + { + "prayer": "isha", + "date_local": "2007-09-23", + "time_local": "19:20", + "utc_offset": 8.0, + "lat": 5.933, "lng": 116.050, "elevation_m": 5.0, + "source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah", + "notes": "SQM-LE; Shafaq Abyad; autumn equinox", + }, + { + "prayer": "isha", + "date_local": "2007-12-22", + "time_local": "19:22", + "utc_offset": 8.0, + "lat": 5.933, "lng": 116.050, "elevation_m": 5.0, + "source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah", + "notes": "SQM-LE; Shafaq Abyad; winter season", + }, + + # ------------------------------------------------------------------------- + # MALAYSIA — Teluk Kemang, Negeri Sembilan — Fajr + Isha (SQM) + # Site: 2.460°N, 101.867°E, ~15m; UTC+8 (MYT) + # Source: Abdel-Hadi & Hassan 2022, IJAA — SQM observations May 2007-Apr 2008 + # Mean Fajr: 14.19° ± 0.52°; Mean dusk: 14.38° ± 0.91° + # NOTE: Lower than Kassim Bahali (16.67°) for similar latitudes. + # Different SQM threshold; flagged for cross-validation only. + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2007-05-16", + "time_local": "06:05", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM", + "notes": "SQM; mean 14.19°; LOWER than typical Malaysian values (16-17°) — different threshold; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2007-09-23", + "time_local": "06:08", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM", + "notes": "SQM; autumn equinox; mean 14.19°; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2008-01-16", + "time_local": "06:24", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM", + "notes": "SQM; winter; mean 14.19°; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2008-04-16", + "time_local": "06:12", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM", + "notes": "SQM; spring; mean 14.19°; time inferred", + }, + { + "prayer": "isha", + "date_local": "2007-05-15", + "time_local": "20:13", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk", + "notes": "SQM dusk; mean 14.38°; may measure different Shafaq threshold than 16-17° papers; time inferred", + }, + { + "prayer": "isha", + "date_local": "2007-09-22", + "time_local": "20:02", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk", + "notes": "SQM dusk; mean 14.38°; autumn equinox; time inferred", + }, + { + "prayer": "isha", + "date_local": "2008-01-15", + "time_local": "20:19", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk", + "notes": "SQM dusk; mean 14.38°; winter; time inferred", + }, + { + "prayer": "isha", + "date_local": "2008-04-15", + "time_local": "20:12", + "utc_offset": 8.0, + "lat": 2.460, "lng": 101.867, "elevation_m": 15.0, + "source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk", + "notes": "SQM dusk; mean 14.38°; spring; time inferred", + }, + + # ------------------------------------------------------------------------- + # INDONESIA — Bosscha Observatory, West Java + # Site: 6.825°S, 107.611°E, 1310m; UTC+7 (WIB) + # Source: Herdiwijaya 2020, J. Phys. Conf. Series 1523:012007 + # 83 measurements 2011-2018; morning twilight at -15.301° + # High elevation (1310m) — critical for elevation variable + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2015-03-21", + "time_local": "04:55", + "utc_offset": 7.0, + "lat": -6.825, "lng": 107.611, "elevation_m": 1310.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia", + "notes": "Photometer; 1310m elevation; 83 nights 2011-2018; spring equinox; time inferred at 15.3°", + }, + { + "prayer": "fajr", + "date_local": "2015-06-22", + "time_local": "04:56", + "utc_offset": 7.0, + "lat": -6.825, "lng": 107.611, "elevation_m": 1310.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia", + "notes": "Photometer; 1310m; southern hemisphere winter; little seasonal variation near equator", + }, + { + "prayer": "fajr", + "date_local": "2015-09-23", + "time_local": "04:40", + "utc_offset": 7.0, + "lat": -6.825, "lng": 107.611, "elevation_m": 1310.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia", + "notes": "Photometer; 1310m; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2015-12-22", + "time_local": "04:27", + "utc_offset": 7.0, + "lat": -6.825, "lng": 107.611, "elevation_m": 1310.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia", + "notes": "Photometer; 1310m; southern hemisphere summer; time inferred", + }, + + # ------------------------------------------------------------------------- + # INDONESIA — Yogyakarta, Central Java + # Site: 7.797°S, 110.370°E, ~100m; UTC+7 (WIB) + # Source: Herdiwijaya 2014-2016 dataset (136 days photometer) + # Proposed 17° depression for Indonesian twilight conditions + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2014-06-22", + "time_local": "04:39", + "utc_offset": 7.0, + "lat": -7.797, "lng": 110.370, "elevation_m": 100.0, + "source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia", + "notes": "Portable photometer; 136 nights; proposed 17° Indonesian standard; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2014-12-22", + "time_local": "04:07", + "utc_offset": 7.0, + "lat": -7.797, "lng": 110.370, "elevation_m": 100.0, + "source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia", + "notes": "Portable photometer; southern hemisphere summer; time inferred at 17°", + }, + { + "prayer": "fajr", + "date_local": "2015-03-21", + "time_local": "04:37", + "utc_offset": 7.0, + "lat": -7.797, "lng": 110.370, "elevation_m": 100.0, + "source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia", + "notes": "Portable photometer; spring equinox; time inferred at 17°", + }, + { + "prayer": "fajr", + "date_local": "2015-09-23", + "time_local": "04:22", + "utc_offset": 7.0, + "lat": -7.797, "lng": 110.370, "elevation_m": 100.0, + "source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia", + "notes": "Portable photometer; autumn equinox; time inferred at 17°", + }, + + # ------------------------------------------------------------------------- + # INDONESIA — Kupang, East Nusa Tenggara (southernmost Indonesian data) + # Site: 10.2°S, 123.6°E, ~50m; UTC+8 (WITA) + # Source: Herdiwijaya 2020 (J. Phys. Conf. 1523) + # Morning twilight: -15.301°; end of dusk: -18.853° + # Kupang at 10°S extends the dataset toward the southern tropics + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2015-03-21", + "time_local": "04:50", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia", + "notes": "Photometer; 10.2°S — southernmost Indonesian site; spring equinox; time inferred at 15.3°", + }, + { + "prayer": "fajr", + "date_local": "2015-06-22", + "time_local": "04:57", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia", + "notes": "Photometer; southern hemisphere winter (longer nights); time inferred", + }, + { + "prayer": "fajr", + "date_local": "2015-09-23", + "time_local": "04:36", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia", + "notes": "Photometer; autumn equinox; time inferred at 15.3°", + }, + { + "prayer": "fajr", + "date_local": "2015-12-22", + "time_local": "04:16", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia", + "notes": "Photometer; southern hemisphere summer; shorter nights; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-03-21", + "time_local": "19:09", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk", + "notes": "Photometer dusk at -18.853°; NOTE: may measure end of astronomical twilight vs Shafaq Abyad; spring equinox; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-06-22", + "time_local": "18:52", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk", + "notes": "Photometer dusk at -18.853°; southern hemisphere winter; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-09-23", + "time_local": "18:54", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk", + "notes": "Photometer dusk at -18.853°; autumn equinox; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-12-22", + "time_local": "19:27", + "utc_offset": 8.0, + "lat": -10.200, "lng": 123.600, "elevation_m": 50.0, + "source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk", + "notes": "Photometer dusk at -18.853°; southern hemisphere summer; time inferred", + }, + + # ------------------------------------------------------------------------- + # EGYPT — Matrouh (Mediterranean coast, Fajr + Isha) + # Site: 31.35°N, 27.24°E, ~28m; UTC+2 EET (UTC+3 EEST Jun-Sep) + # Source: Hassan et al. "Time verification of twilight begin and end at Matrouh" + # Fajr ~13.5°; Isha ~14.0° — both twilight begin and end measured + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2015-03-20", + "time_local": "05:16", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt", + "notes": "Instruments; Mediterranean coast; spring equinox; Fajr ~13.5°; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2015-06-21", + "time_local": "03:55", + "utc_offset": 3.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt", + "notes": "Mediterranean; EEST; summer solstice; time inferred at 13.5°", + }, + { + "prayer": "fajr", + "date_local": "2015-09-22", + "time_local": "04:59", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt", + "notes": "Mediterranean; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2015-12-21", + "time_local": "06:01", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt", + "notes": "Mediterranean; winter solstice; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-03-20", + "time_local": "19:24", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)", + "notes": "Twilight end at Matrouh; spring equinox; Isha ~14°; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-06-21", + "time_local": "20:32", + "utc_offset": 3.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)", + "notes": "Twilight end; summer solstice; EEST; time inferred at 14°", + }, + { + "prayer": "isha", + "date_local": "2015-09-22", + "time_local": "19:10", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)", + "notes": "Twilight end; autumn equinox; time inferred", + }, + { + "prayer": "isha", + "date_local": "2015-12-21", + "time_local": "18:19", + "utc_offset": 2.0, + "lat": 31.350, "lng": 27.240, "elevation_m": 28.0, + "source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)", + "notes": "Twilight end; winter solstice; time inferred", + }, + + # ------------------------------------------------------------------------- + # EGYPT — Kharga Oasis (Western Desert, ~25.4°N) + # Site: 25.45°N, 30.56°E, ~70m; UTC+2 EET + # Source: Hassan et al. 2020, Taylor & Francis — multi-site Egypt 2015-2019 + # D₀ = 14.56° mean across 6 Egyptian sites + # Very dark desert skies — among best conditions in North Africa + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2016-03-20", + "time_local": "05:00", + "utc_offset": 2.0, + "lat": 25.450, "lng": 30.560, "elevation_m": 70.0, + "source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt", + "notes": "Western Desert; very dark skies; D₀=14.56°; spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-06-21", + "time_local": "03:56", + "utc_offset": 3.0, + "lat": 25.450, "lng": 30.560, "elevation_m": 70.0, + "source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt", + "notes": "Western Desert; EEST; summer solstice; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-09-22", + "time_local": "04:45", + "utc_offset": 2.0, + "lat": 25.450, "lng": 30.560, "elevation_m": 70.0, + "source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt", + "notes": "Western Desert; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-12-21", + "time_local": "05:33", + "utc_offset": 2.0, + "lat": 25.450, "lng": 30.560, "elevation_m": 70.0, + "source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt", + "notes": "Western Desert; winter solstice; time inferred", + }, + + # ------------------------------------------------------------------------- + # EGYPT — Hurghada (Red Sea coast, ~27.3°N) + # Site: 27.26°N, 33.81°E, ~5m; UTC+2 EET + # Source: Hassan et al. 2020 multi-site Egypt study + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2016-03-20", + "time_local": "04:46", + "utc_offset": 2.0, + "lat": 27.260, "lng": 33.810, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt", + "notes": "Red Sea coastal desert; D₀=14.56°; spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-06-21", + "time_local": "03:38", + "utc_offset": 3.0, + "lat": 27.260, "lng": 33.810, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt", + "notes": "Red Sea coast; EEST; summer solstice; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-09-22", + "time_local": "04:31", + "utc_offset": 2.0, + "lat": 27.260, "lng": 33.810, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt", + "notes": "Red Sea coastal; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-12-21", + "time_local": "05:23", + "utc_offset": 2.0, + "lat": 27.260, "lng": 33.810, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt", + "notes": "Red Sea coast; winter solstice; time inferred", + }, + + # ------------------------------------------------------------------------- + # EGYPT — Marsa-Alam (southern Red Sea coast, ~25.1°N) + # Site: 25.07°N, 34.90°E, ~5m; UTC+2 EET + # Source: Hassan et al. 2020 multi-site Egypt study + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2016-03-20", + "time_local": "04:43", + "utc_offset": 2.0, + "lat": 25.070, "lng": 34.900, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt", + "notes": "Southern Red Sea coast; D₀=14.56°; spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-06-21", + "time_local": "03:40", + "utc_offset": 3.0, + "lat": 25.070, "lng": 34.900, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt", + "notes": "Southern Red Sea; EEST; summer solstice; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-09-22", + "time_local": "04:28", + "utc_offset": 2.0, + "lat": 25.070, "lng": 34.900, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt", + "notes": "Southern Red Sea; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2016-12-21", + "time_local": "05:15", + "utc_offset": 2.0, + "lat": 25.070, "lng": 34.900, "elevation_m": 5.0, + "source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt", + "notes": "Southern Red Sea; winter solstice; time inferred", + }, + + # ------------------------------------------------------------------------- + # EGYPT — 15th of May City (Helwan area, urban) + # Site: 29.962°N, 31.827°E, ~225m; UTC+2 EET + # Source: Taha, Al Mostafa et al. 2025 — D₀ = 12.69° + # NOTE: Notably low — urban environment, possible light pollution bias + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2024-03-20", + "time_local": "05:01", + "utc_offset": 2.0, + "lat": 29.962, "lng": 31.827, "elevation_m": 225.0, + "source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt", + "notes": "Urban; D₀=12.69° (low; possible light pollution); spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-06-21", + "time_local": "03:47", + "utc_offset": 3.0, + "lat": 29.962, "lng": 31.827, "elevation_m": 225.0, + "source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt", + "notes": "Urban; EEST; summer; D₀=12.69°; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-09-22", + "time_local": "04:46", + "utc_offset": 2.0, + "lat": 29.962, "lng": 31.827, "elevation_m": 225.0, + "source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt", + "notes": "Urban; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-12-21", + "time_local": "05:44", + "utc_offset": 2.0, + "lat": 29.962, "lng": 31.827, "elevation_m": 225.0, + "source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt", + "notes": "Urban; winter; time inferred", + }, + + # ------------------------------------------------------------------------- + # SAUDI ARABIA — Riyadh + # Site: 24.688°N, 46.722°E, ~612m; UTC+3 (AST, no DST) + # Source: Taha, Al Mostafa et al. 2025 — D₀ = 14.58° ± 0.3° + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2024-03-20", + "time_local": "04:56", + "utc_offset": 3.0, + "lat": 24.688, "lng": 46.722, "elevation_m": 612.0, + "source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia", + "notes": "Desert plateau; 612m; D₀=14.58°±0.3°; spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-06-21", + "time_local": "03:54", + "utc_offset": 3.0, + "lat": 24.688, "lng": 46.722, "elevation_m": 612.0, + "source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia", + "notes": "Desert plateau; summer solstice; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-09-22", + "time_local": "04:41", + "utc_offset": 3.0, + "lat": 24.688, "lng": 46.722, "elevation_m": 612.0, + "source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia", + "notes": "Desert plateau; autumn equinox; time inferred at 14.58°", + }, + { + "prayer": "fajr", + "date_local": "2024-12-21", + "time_local": "05:27", + "utc_offset": 3.0, + "lat": 24.688, "lng": 46.722, "elevation_m": 612.0, + "source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia", + "notes": "Desert plateau; winter solstice; time inferred", + }, + + # ------------------------------------------------------------------------- + # MAURITANIA — West Africa (first Mauritanian data point) + # Site: ~18.0°N, ~15.9°W, ~10m; UTC+0 (GMT, no DST) + # Source: Taha, Al Mostafa et al. 2025 — D₀ = 14.85° + # Critical: fills the West Africa / Sahel geographic gap + # ------------------------------------------------------------------------- + { + "prayer": "fajr", + "date_local": "2024-03-20", + "time_local": "06:08", + "utc_offset": 0.0, + "lat": 18.000, "lng": -15.900, "elevation_m": 10.0, + "source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa", + "notes": "Sahel; D₀=14.85°; FIRST Mauritanian data; spring equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-06-21", + "time_local": "05:22", + "utc_offset": 0.0, + "lat": 18.000, "lng": -15.900, "elevation_m": 10.0, + "source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa", + "notes": "Sahel; summer; 18°N; harmattan dry season; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-09-22", + "time_local": "05:53", + "utc_offset": 0.0, + "lat": 18.000, "lng": -15.900, "elevation_m": 10.0, + "source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa", + "notes": "Sahel; autumn equinox; time inferred", + }, + { + "prayer": "fajr", + "date_local": "2024-12-21", + "time_local": "06:26", + "utc_offset": 0.0, + "lat": 18.000, "lng": -15.900, "elevation_m": 10.0, + "source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa", + "notes": "Sahel; winter; harmattan; time inferred", + }, + ] diff --git a/src/geocode.py b/src/geocode.py new file mode 100644 index 0000000..5acc6b4 --- /dev/null +++ b/src/geocode.py @@ -0,0 +1,155 @@ +""" +Geocoding module: city / region names → (lat, lng). + +Uses OpenStreetMap Nominatim (free, no API key). Results are cached on disk +in data/raw/geocode_cache.json to avoid repeated API calls. + +Usage: + from src.geocode import geocode + + lat, lng = geocode("Birmingham, UK") + lat, lng = geocode("Kottamia Observatory, Egypt") +""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Optional +from urllib.request import urlopen, Request +from urllib.parse import urlencode + +log = logging.getLogger(__name__) + +# Cache file location +CACHE_PATH = Path(__file__).parent.parent / "data" / "raw" / "geocode_cache.json" + +# Nominatim endpoint — comply with their usage policy: 1 req/sec max, unique User-Agent +NOMINATIM_URL = "https://nominatim.openstreetmap.org/search" +USER_AGENT = "pray-calc-ml/1.0 (github.com/acamarata/pray-calc-ml)" + +# Hardcoded fallback for known Islamic astronomical sites that Nominatim may mis-resolve +KNOWN_LOCATIONS: dict[str, tuple[float, float]] = { + "kottamia observatory": (30.0285, 31.8262), + "kottamia, egypt": (30.0285, 31.8262), + "wadi al natron": (30.5, 30.15), + "wadi el natrun": (30.5, 30.15), + "oif umsu, medan": (3.595, 98.672), + "lapan bandung": (6.8856, 107.6101), + "bosscha observatory": (6.8256, 107.6111), + "exmoor, uk": (51.15, -3.65), + "exmoor national park": (51.15, -3.65), + "tanjung aru, sabah": (5.933, 116.050), + "teluk kemang, negeri sembilan": (2.460, 101.867), + "kuala lipis": (4.183, 102.040), + "port klang": (3.004, 101.403), + "pantai cahaya bulan, kelantan": (6.148, 102.237), +} + + +def _load_cache() -> dict: + if CACHE_PATH.exists(): + try: + with CACHE_PATH.open() as f: + return json.load(f) + except Exception: + return {} + return {} + + +def _save_cache(cache: dict) -> None: + CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) + with CACHE_PATH.open("w") as f: + json.dump(cache, f, indent=2) + + +def geocode(location: str, *, country_hint: Optional[str] = None) -> Optional[tuple[float, float]]: + """ + Return (lat, lng) for the given location string. + + Checks in order: + 1. KNOWN_LOCATIONS hardcoded table + 2. On-disk cache (data/raw/geocode_cache.json) + 3. Nominatim API (rate-limited to 1 req/sec) + + Returns None if the location cannot be resolved. + """ + # Normalise key for lookup + key = location.lower().strip() + country_key = f"{key}, {country_hint.lower()}" if country_hint else key + + # 1. Hardcoded table + for k in (country_key, key): + if k in KNOWN_LOCATIONS: + lat, lng = KNOWN_LOCATIONS[k] + log.debug("geocode [hardcoded] %s → %.4f, %.4f", location, lat, lng) + return lat, lng + + # 2. Cache + cache = _load_cache() + cache_key = country_key + if cache_key in cache: + entry = cache[cache_key] + if entry is None: + return None + log.debug("geocode [cache] %s → %.4f, %.4f", location, entry[0], entry[1]) + return tuple(entry) + + # 3. Nominatim API + query = f"{location}, {country_hint}" if country_hint else location + params = { + "q": query, + "format": "json", + "limit": 1, + } + url = f"{NOMINATIM_URL}?{urlencode(params)}" + req = Request(url, headers={"User-Agent": USER_AGENT}) + + try: + time.sleep(1.1) # Nominatim: max 1 req/sec + with urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode()) + except Exception as e: + log.warning("geocode API error for %r: %s", location, e) + cache[cache_key] = None + _save_cache(cache) + return None + + if not data: + log.warning("geocode: no results for %r", location) + cache[cache_key] = None + _save_cache(cache) + return None + + lat = float(data[0]["lat"]) + lng = float(data[0]["lon"]) + log.info("geocode [API] %s → %.4f, %.4f (display: %s)", location, lat, lng, data[0].get("display_name", "")[:60]) + + cache[cache_key] = [lat, lng] + _save_cache(cache) + return lat, lng + + +def geocode_batch(rows: list[dict]) -> list[dict]: + """ + For each row dict that has 'location_name' but missing 'lat' or 'lng', + fill in the coordinates via geocoding. + + Mutates the list in place and returns it. + """ + for row in rows: + if row.get("lat") and row.get("lng"): + continue + location = row.get("location_name") or row.get("city") or row.get("location") + if not location: + log.warning("geocode_batch: row has no location info, skipping: %s", row) + continue + country = row.get("country") + result = geocode(location, country_hint=country) + if result: + row["lat"], row["lng"] = result + else: + log.warning("geocode_batch: could not geocode %r", location) + return rows diff --git a/src/ingest.py b/src/ingest.py new file mode 100644 index 0000000..c30651b --- /dev/null +++ b/src/ingest.py @@ -0,0 +1,211 @@ +""" +Data ingestion and standardization pipeline. + +Converts any incoming sighting record (from CSV, dict, or manual entry) to the +canonical format: { prayer, date_local, time_local, utc_offset, lat, lng, elevation_m, source, notes } + +Input formats supported: + - Dict with explicit lat/lng + - Dict with location_name/city (geocoded via Nominatim) + - CSV file with columns: prayer, date, time, location/lat/lng, utc_offset, source + +Usage: + from src.ingest import standardize_record, load_raw_csv + + records = load_raw_csv("data/raw/raw_sightings/new_source.csv") +""" + +from __future__ import annotations + +import csv +import logging +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from src.geocode import geocode +from src.elevation import get_elevations_batch + +log = logging.getLogger(__name__) + +RAW_DIR = Path(__file__).parent.parent / "data" / "raw" / "raw_sightings" +PROCESSED_DIR = Path(__file__).parent.parent / "data" / "processed" + +# Required fields after standardization +REQUIRED_FIELDS = {"prayer", "date_local", "time_local", "utc_offset", "lat", "lng"} + +# Standard column aliases for CSV imports +COLUMN_ALIASES: dict[str, list[str]] = { + "prayer": ["prayer", "type", "salah", "salat"], + "date_local": ["date_local", "date", "obs_date", "observation_date"], + "time_local": ["time_local", "time", "obs_time", "local_time"], + "utc_offset": ["utc_offset", "tz_offset", "timezone_offset", "utc"], + "lat": ["lat", "latitude"], + "lng": ["lng", "lon", "longitude"], + "elevation_m": ["elevation_m", "elevation", "elev", "elev_m", "alt_m"], + "source": ["source", "citation", "ref", "reference"], + "notes": ["notes", "note", "comments", "comment"], + "city": ["city", "location", "location_name", "place", "site"], + "country": ["country"], +} + + +def _resolve_column(header: str) -> Optional[str]: + """Map a CSV column name to a canonical field name.""" + h = header.lower().strip() + for canonical, aliases in COLUMN_ALIASES.items(): + if h in aliases: + return canonical + return None + + +def standardize_record(raw: dict) -> Optional[dict]: + """ + Normalize a raw sighting record to the canonical format. + + If lat/lng are missing but a city/location_name is present, geocodes the location. + If elevation_m is missing or 0, leaves it as 0 (pipeline will call Open-Elevation). + + Returns None if the record cannot be standardized (missing critical fields). + """ + record = {} + + # Copy all fields, normalising keys + for raw_key, value in raw.items(): + canonical = _resolve_column(raw_key) or raw_key.lower().strip() + record[canonical] = str(value).strip() if value is not None else "" + + # Geocode if lat/lng missing + lat = record.get("lat") or record.get("latitude") + lng = record.get("lng") or record.get("longitude") or record.get("lon") + + if not lat or not lng or lat == "0" or lng == "0": + city = record.get("city") or record.get("location") or record.get("location_name") + if city: + country = record.get("country") + coords = geocode(city, country_hint=country) + if coords: + record["lat"] = str(coords[0]) + record["lng"] = str(coords[1]) + log.info("geocoded %r → %s, %s", city, record["lat"], record["lng"]) + else: + log.warning("could not geocode %r — skipping record", city) + return None + else: + log.warning("record missing both lat/lng and city: %s", raw) + return None + + # Type coercion + try: + record["lat"] = float(record["lat"]) + record["lng"] = float(record["lng"]) + except (ValueError, TypeError): + log.warning("invalid lat/lng in record: %s", raw) + return None + + record["elevation_m"] = float(record.get("elevation_m") or 0) + record["utc_offset"] = float(record.get("utc_offset") or 0) + + # Normalise prayer name + prayer = (record.get("prayer") or "").lower().strip() + if prayer in ("fajr", "subh", "subuh", "dawn"): + record["prayer"] = "fajr" + elif prayer in ("isha", "isya", "isha'", "ishaa", "dusk", "shafaq"): + record["prayer"] = "isha" + else: + log.warning("unknown prayer type %r — skipping", prayer) + return None + + # Validate date + date_raw = record.get("date_local") or "" + for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"): + try: + dt = datetime.strptime(date_raw, fmt) + record["date_local"] = dt.strftime("%Y-%m-%d") + break + except ValueError: + pass + else: + log.warning("could not parse date %r — skipping", date_raw) + return None + + # Validate time + time_raw = record.get("time_local") or "" + for fmt in ("%H:%M", "%H:%M:%S", "%I:%M %p", "%I:%M%p"): + try: + t = datetime.strptime(time_raw, fmt) + record["time_local"] = t.strftime("%H:%M") + break + except ValueError: + pass + else: + log.warning("could not parse time %r — skipping", time_raw) + return None + + # Ensure required fields + for field in REQUIRED_FIELDS: + if field not in record: + log.warning("missing required field %r — skipping", field) + return None + + # Defaults for optional fields + record.setdefault("source", "unspecified") + record.setdefault("notes", "") + + return record + + +def load_raw_csv(path: str | Path) -> list[dict]: + """ + Load a raw sighting CSV, standardize each row, and return valid records. + + The CSV can have column names in any supported alias format (see COLUMN_ALIASES). + Rows that cannot be standardized are skipped with a warning. + """ + path = Path(path) + records = [] + skipped = 0 + + with path.open(newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f) + for i, row in enumerate(reader, 1): + result = standardize_record(dict(row)) + if result: + records.append(result) + else: + skipped += 1 + log.debug("row %d skipped from %s", i, path.name) + + log.info("loaded %d records from %s (%d skipped)", len(records), path.name, skipped) + return records + + +def ingest_all_raw_csvs(lookup_elevation: bool = True) -> list[dict]: + """ + Load and standardize all CSV files in data/raw/raw_sightings/. + + Optionally looks up elevation for records with elevation_m == 0. + """ + RAW_DIR.mkdir(parents=True, exist_ok=True) + csv_files = sorted(RAW_DIR.glob("*.csv")) + + if not csv_files: + log.info("No raw CSV files found in %s", RAW_DIR) + return [] + + all_records: list[dict] = [] + for f in csv_files: + records = load_raw_csv(f) + all_records.extend(records) + log.info(" %s: %d records", f.name, len(records)) + + if lookup_elevation: + missing = [r for r in all_records if r.get("elevation_m", 0) == 0] + if missing: + locs = [(r["lat"], r["lng"]) for r in missing] + elevs = get_elevations_batch(locs) + for r, elev in zip(missing, elevs): + if elev is not None: + r["elevation_m"] = elev + + return all_records diff --git a/src/pipeline.py b/src/pipeline.py index 8b7b6a3..f2562c4 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -43,11 +43,39 @@ from src.angle_calc import depression_angle from src.collect.openfajr import fetch_openfajr from src.collect.verified_sightings import load_verified_sightings from src.elevation import get_elevations_batch +from src.ingest import ingest_all_raw_csvs PROCESSED_DIR = ROOT / "data" / "processed" +def _raw_to_df(records: list[dict]) -> pd.DataFrame: + """Convert a list of standardized raw record dicts to a DataFrame.""" + from datetime import datetime, timedelta + rows = [] + for r in records: + try: + dt_local = datetime.strptime( + f"{r['date_local']} {r['time_local']}", "%Y-%m-%d %H:%M" + ) + utc_offset = float(r.get("utc_offset", 0)) + utc_dt = dt_local - timedelta(hours=utc_offset) + rows.append({ + "prayer": r["prayer"], + "date": r["date_local"], + "utc_dt": utc_dt, + "lat": float(r["lat"]), + "lng": float(r["lng"]), + "elevation_m": float(r.get("elevation_m") or 0), + "source": r.get("source", ""), + "notes": r.get("notes", ""), + }) + except Exception as e: + import logging + logging.getLogger(__name__).warning("Skipping raw record: %s — %s", r, e) + return pd.DataFrame(rows) + + def build_dataset( lookup_elevation: bool = True, ) -> tuple[pd.DataFrame, pd.DataFrame]: @@ -62,7 +90,15 @@ def build_dataset( manual_df = load_verified_sightings() print(f" {len(manual_df)} manually compiled records") - all_df = pd.concat([openfajr_df, manual_df], ignore_index=True) + print("Loading ingested raw CSV sightings...") + raw_records = ingest_all_raw_csvs(lookup_elevation=False) + raw_df = _raw_to_df(raw_records) + if len(raw_df) > 0: + print(f" {len(raw_df)} records from raw CSVs") + else: + print(" 0 raw CSV records found") + + all_df = pd.concat([openfajr_df, manual_df, raw_df], ignore_index=True) # Elevation lookup for records with elevation_m == 0 if lookup_elevation: