mirror of
https://github.com/acamarata/pray-calc-ml.git
synced 2026-06-30 19:04:26 +00:00
Expand dataset to 4,149 Fajr / 58 Isha records across 46 locations
New records from research expansion: - Tanjung Aru, Sabah Malaysia (Niri & Zainuddin): 4 Isha Shafaq Abyad records - Teluk Kemang, Malaysia (Abdel-Hadi & Hassan 2022): 4 Fajr + 4 Isha SQM records - Bosscha Observatory, Java 1310m (Herdiwijaya 2020): 4 Fajr records - Yogyakarta, Java (Herdiwijaya 2014-2016, 136 nights): 4 Fajr records - Kupang, NTT 10°S (Herdiwijaya 2020): 4 Fajr + 4 Isha records - Matrouh, Egypt (Hassan et al.): 4 Fajr + 3 Isha records (1 filtered) - Kharga Oasis, Egypt (Hassan et al. 2020): 4 Fajr records - Hurghada, Egypt (Hassan et al. 2020): 4 Fajr records - Marsa-Alam, Egypt (Hassan et al. 2020): 4 Fajr records - 15th of May City, Egypt (Taha et al. 2025): 4 Fajr records - Riyadh, Saudi Arabia (Taha et al. 2025): 4 Fajr records - Mauritania 18°N (Taha et al. 2025): 4 Fajr records — first West Africa data New modules: - src/geocode.py: Nominatim geocoding with disk cache - src/ingest.py: CSV ingestion and data standardization pipeline - src/pipeline.py: integrated raw CSV loading via ingest module
This commit is contained in:
Aric Camarata
parent
a5b8adfb2d
commit
0f01783516
6 changed files with 1093 additions and 1 deletions
|
|
@ -6,13 +6,29 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes
|
|||
2007-12-21,2007-12-20 15:38:00+00:00,-36.87,174.76,20.0,354,13.55718784204495,"Moonsighting.com / Khalid Shaukat, Auckland New Zealand",Southern hemisphere summer
|
||||
2006-06-21,2006-06-21 04:05:00+00:00,-33.93,18.42,10.0,172,21.34857245848067,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Southern hemisphere winter; 33°S latitude
|
||||
2006-12-21,2006-12-21 02:10:00+00:00,-33.93,18.42,10.0,355,14.614263776489265,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Southern hemisphere summer; seasons are reversed
|
||||
2015-03-21,2015-03-20 20:50:00+00:00,-10.2,123.6,50.0,79,15.501549745903526,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; 10.2°S — southernmost Indonesian site; spring equinox; time inferred at 15.3°
|
||||
2015-06-22,2015-06-21 20:57:00+00:00,-10.2,123.6,50.0,172,15.51337451090411,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; southern hemisphere winter (longer nights); time inferred
|
||||
2015-09-23,2015-09-22 20:36:00+00:00,-10.2,123.6,50.0,265,15.35923592435714,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; autumn equinox; time inferred at 15.3°
|
||||
2015-12-22,2015-12-21 20:16:00+00:00,-10.2,123.6,50.0,355,15.473409366523073,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",Photometer; southern hemisphere summer; shorter nights; time inferred
|
||||
2015-03-21,2015-03-20 21:37:00+00:00,-7.797,110.37,100.0,79,17.073403377713362,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; spring equinox; time inferred at 17°
|
||||
2014-06-22,2014-06-21 21:39:00+00:00,-7.797,110.37,100.0,172,17.128376155248212,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; 136 nights; proposed 17° Indonesian standard; time inferred
|
||||
2015-09-23,2015-09-22 21:22:00+00:00,-7.797,110.37,100.0,265,17.164007544600874,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; autumn equinox; time inferred at 17°
|
||||
2014-12-22,2014-12-21 21:07:00+00:00,-7.797,110.37,100.0,355,17.03760294117874,"Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",Portable photometer; southern hemisphere summer; time inferred at 17°
|
||||
2011-06-21,2011-06-20 21:33:00+00:00,-7.55,112.23,44.0,171,16.64428137518168,"Bandung/Jombang study 2012, AIP Conf. Proc. 1454",SQM observation; Jombang lowland site
|
||||
2011-06-21,2011-06-20 21:28:00+00:00,-6.914,107.609,768.0,171,21.78801146175366,"Bandung/Jombang study 2012, AIP Conf. Proc. 1454",SQM observation; Bandung highland site 768m
|
||||
2015-03-21,2015-03-20 21:55:00+00:00,-6.825,107.611,1310.0,79,15.38420880351,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m elevation; 83 nights 2011-2018; spring equinox; time inferred at 15.3°
|
||||
2015-06-22,2015-06-21 21:56:00+00:00,-6.825,107.611,1310.0,172,15.382149501475242,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; southern hemisphere winter; little seasonal variation near equator
|
||||
2015-09-23,2015-09-22 21:40:00+00:00,-6.825,107.611,1310.0,265,15.469917432309906,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; autumn equinox; time inferred
|
||||
2015-12-22,2015-12-21 21:27:00+00:00,-6.825,107.611,1310.0,355,15.479969763220872,"Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",Photometer; 1310m; southern hemisphere summer; time inferred
|
||||
2015-06-01,2015-05-31 21:37:00+00:00,-6.4,106.83,65.0,151,19.376604637544194,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM confirmed; near equator observation
|
||||
2015-06-21,2015-06-20 21:38:00+00:00,-6.4,106.83,65.0,171,20.013330239898416,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM sky brightness confirmed Fajr; southern hemisphere
|
||||
2015-07-15,2015-07-14 21:40:00+00:00,-6.4,106.83,65.0,195,20.58680999808561,"Saksono 2020, NRIAG J. 9(1):238-244, Depok Indonesia",SQM confirmed; winter in southern hemisphere
|
||||
2015-06-21,2015-06-21 02:02:00+00:00,-4.05,39.67,50.0,172,20.164495986609136,"Community observations, Mombasa Kenya (2012-2016)",Near equatorial; 4°S; Indian Ocean coastal Kenya
|
||||
2015-12-21,2015-12-21 01:45:00+00:00,-4.05,39.67,50.0,355,19.6971437077456,"Community observations, Mombasa Kenya (2012-2016)",Southern hemisphere summer; near equator
|
||||
2008-01-16,2008-01-15 22:24:00+00:00,2.46,101.867,15.0,15,14.394345345593194,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; winter; mean 14.19°; time inferred
|
||||
2008-04-16,2008-04-15 22:12:00+00:00,2.46,101.867,15.0,106,14.395231562638005,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; spring; mean 14.19°; time inferred
|
||||
2007-05-16,2007-05-15 22:05:00+00:00,2.46,101.867,15.0,135,14.2440273961132,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; mean 14.19°; LOWER than typical Malaysian values (16-17°) — different threshold; time inferred
|
||||
2007-09-23,2007-09-22 22:08:00+00:00,2.46,101.867,15.0,265,14.285295399078418,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",SQM; autumn equinox; mean 14.19°; time inferred
|
||||
2017-03-20,2017-03-19 22:00:00+00:00,3.14,101.69,40.0,78,20.184283044634665,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",Spring equinox; near equator
|
||||
2017-06-21,2017-06-20 21:57:00+00:00,3.14,101.69,40.0,171,16.488023722444712,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",DSLR + SQM confirmed; mean depression ~16.67° across 64 days
|
||||
2017-09-22,2017-09-21 22:00:00+00:00,3.14,101.69,40.0,264,16.477962853797283,"Kassim Bahali 2018, Sains Malaysia 47(11), Kuala Lumpur",Autumn equinox; near equator
|
||||
|
|
@ -29,6 +45,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes
|
|||
2013-12-21,2013-12-21 04:55:00+00:00,11.99,8.51,476.0,355,11.291542377365435,"Community observations, Kano Nigeria (2010-2015)",Sahelian winter; 12°N; dry harmattan season
|
||||
2016-06-21,2016-06-21 05:22:00+00:00,14.72,-17.47,24.0,173,17.919330844155905,"Community observations, Dakar Senegal (2015-2018)",Summer; Dakar; hot season in West Africa
|
||||
2016-12-21,2016-12-21 06:08:00+00:00,14.72,-17.47,24.0,356,19.325677039545287,"Community observations, Dakar Senegal (2015-2018)",Winter; Sahel; 14.7°N latitude; coastal low elevation
|
||||
2024-03-20,2024-03-20 06:08:00+00:00,18.0,-15.9,10.0,80,14.942669517067234,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; D₀=14.85°; FIRST Mauritanian data; spring equinox; time inferred
|
||||
2024-06-21,2024-06-21 05:22:00+00:00,18.0,-15.9,10.0,173,14.93961129986636,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; summer; 18°N; harmattan dry season; time inferred
|
||||
2024-09-22,2024-09-22 05:53:00+00:00,18.0,-15.9,10.0,266,14.978505812094998,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; autumn equinox; time inferred
|
||||
2024-12-21,2024-12-21 06:26:00+00:00,18.0,-15.9,10.0,356,14.987994652399438,"Taha et al. 2025, Emirates Scholar, Mauritania West Africa",Sahel; winter; harmattan; time inferred
|
||||
2014-06-21,2014-06-21 00:10:00+00:00,23.61,58.59,9.0,172,14.62507938103998,"Oman Ministry of Awqaf, Muscat observations",Summer; very hot; coastal Arabia
|
||||
2014-12-21,2014-12-21 01:42:00+00:00,23.61,58.59,9.0,355,13.762380385090472,"Oman Ministry of Awqaf, Muscat observations",Winter; Arabian coastal desert; 23.6°N
|
||||
2014-03-20,2014-03-19 22:38:00+00:00,23.71,90.41,8.0,78,20.18939031858654,"Bangladesh Islamic Foundation, Dhaka observations",Spring equinox; Dhaka tropical delta
|
||||
|
|
@ -37,13 +57,29 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes
|
|||
2014-12-21,2014-12-20 23:22:00+00:00,23.71,90.41,8.0,354,16.533038084169682,"Bangladesh Islamic Foundation, Dhaka observations",Winter; tropical flat delta; 23.7°N
|
||||
1987-06-21,1987-06-21 00:50:00+00:00,24.09,32.9,92.0,172,25.449771498445873,"Hassan et al. 2014, NRIAG J. 3:23-26, Aswan Egypt",Summer solstice; Aswan desert
|
||||
1986-12-21,1986-12-21 03:36:00+00:00,24.09,32.9,92.0,355,11.518719591801936,"Hassan et al. 2014, NRIAG J. 3:23-26, Aswan Egypt",Desert; near Tropic of Cancer; 1984-1987 study; time inferred from published mean angle
|
||||
2024-03-20,2024-03-20 01:56:00+00:00,24.688,46.722,612.0,80,14.637419825322901,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; 612m; D₀=14.58°±0.3°; spring equinox; time inferred
|
||||
2024-06-21,2024-06-21 00:54:00+00:00,24.688,46.722,612.0,173,14.69394817844122,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; summer solstice; time inferred
|
||||
2024-09-22,2024-09-22 01:41:00+00:00,24.688,46.722,612.0,266,14.6067164174564,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; autumn equinox; time inferred at 14.58°
|
||||
2024-12-21,2024-12-21 02:27:00+00:00,24.688,46.722,612.0,356,14.717739434867665,"Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",Desert plateau; winter solstice; time inferred
|
||||
2005-06-21,2005-06-20 23:05:00+00:00,24.86,67.01,8.0,171,19.608139512671162,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Summer; near 25°N latitude
|
||||
2005-12-21,2005-12-21 00:40:00+00:00,24.86,67.01,8.0,355,20.305030543876732,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Winter; 15°-16° documented for Karachi across seasons
|
||||
2016-03-20,2016-03-20 02:43:00+00:00,25.07,34.9,5.0,80,14.665780478304086,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea coast; D₀=14.56°; spring equinox; time inferred
|
||||
2016-06-21,2016-06-21 00:40:00+00:00,25.07,34.9,5.0,173,25.1323452869587,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; EEST; summer solstice; time inferred
|
||||
2016-09-22,2016-09-22 02:28:00+00:00,25.07,34.9,5.0,266,14.61683192761719,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; autumn equinox; time inferred
|
||||
2016-12-21,2016-12-21 03:15:00+00:00,25.07,34.9,5.0,356,14.693094404794968,"Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",Southern Red Sea; winter solstice; time inferred
|
||||
2016-06-21,2016-06-20 23:48:00+00:00,25.2,55.27,11.0,172,20.106143972982427,"Dubai Awqaf / GSMC observations, Dubai UAE",Summer; very hot desert; Dubai
|
||||
2016-09-22,2016-09-22 00:52:00+00:00,25.2,55.27,11.0,266,17.84568432943024,"Dubai Awqaf / GSMC observations, Dubai UAE",Autumn equinox; Dubai desert
|
||||
2016-12-21,2016-12-21 01:52:00+00:00,25.2,55.27,11.0,356,15.05455996850309,"Dubai Awqaf / GSMC observations, Dubai UAE",Winter; desert coastal; 25°N
|
||||
2016-03-20,2016-03-20 03:00:00+00:00,25.45,30.56,70.0,80,14.69695004658144,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; very dark skies; D₀=14.56°; spring equinox; time inferred
|
||||
2016-06-21,2016-06-21 00:56:00+00:00,25.45,30.56,70.0,173,25.0817352927562,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; EEST; summer solstice; time inferred
|
||||
2016-09-22,2016-09-22 02:45:00+00:00,25.45,30.56,70.0,266,14.650410845124265,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; autumn equinox; time inferred
|
||||
2016-12-21,2016-12-21 03:33:00+00:00,25.45,30.56,70.0,356,14.683638007889085,"Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",Western Desert; winter solstice; time inferred
|
||||
2013-06-21,2013-06-21 00:48:00+00:00,27.17,31.17,55.0,172,24.701372303161943,"Hassan et al. 2016, NRIAG J. 5:9-15, Assiut Egypt",Summer solstice; Nile Valley site
|
||||
2012-12-21,2012-12-21 03:22:00+00:00,27.17,31.17,55.0,356,17.055260687730517,"Hassan et al. 2016, NRIAG J. 5:9-15, Assiut Egypt",Naked eye; Nile Valley; published mean depression 13.665°; time inferred
|
||||
2016-03-20,2016-03-20 02:46:00+00:00,27.26,33.81,5.0,80,14.687319479851721,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coastal desert; D₀=14.56°; spring equinox; time inferred
|
||||
2016-06-21,2016-06-21 00:38:00+00:00,27.26,33.81,5.0,173,24.560665283617805,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coast; EEST; summer solstice; time inferred
|
||||
2016-09-22,2016-09-22 02:31:00+00:00,27.26,33.81,5.0,266,14.630253249336265,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coastal; autumn equinox; time inferred
|
||||
2016-12-21,2016-12-21 03:23:00+00:00,27.26,33.81,5.0,356,14.644130486182439,"Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",Red Sea coast; winter solstice; time inferred
|
||||
2015-01-15,2015-01-15 03:08:00+00:00,27.52,41.7,1020.0,15,12.64172648853074,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Winter; desert plateau ~1000m elevation
|
||||
2015-03-20,2015-03-20 02:18:00+00:00,27.52,41.7,1020.0,79,14.063019699145046,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Spring equinox; Hail desert
|
||||
2015-06-21,2015-06-21 00:38:00+00:00,27.52,41.7,1020.0,172,19.30662081176095,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Summer solstice
|
||||
|
|
@ -52,6 +88,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes
|
|||
2019-06-21,2019-06-21 00:52:00+00:00,29.28,30.05,50.0,172,23.219431281586154,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Summer solstice
|
||||
2018-09-22,2018-09-22 02:50:00+00:00,29.28,30.05,50.0,265,13.43585038521332,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Autumn equinox
|
||||
2018-12-21,2018-12-21 04:08:00+00:00,29.28,30.05,50.0,355,9.12258763189756,"Rashed et al. 2022, IJMET 13(10), Fayum Egypt",Winter naked-eye + SQM confirmed Fajr
|
||||
2024-03-20,2024-03-20 03:01:00+00:00,29.962,31.827,225.0,80,12.770050375109646,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; D₀=12.69° (low; possible light pollution); spring equinox; time inferred
|
||||
2024-06-21,2024-06-21 00:47:00+00:00,29.962,31.827,225.0,173,22.430622979625138,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; EEST; summer; D₀=12.69°; time inferred
|
||||
2024-09-22,2024-09-22 02:46:00+00:00,29.962,31.827,225.0,266,12.73836427377631,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; autumn equinox; time inferred
|
||||
2024-12-21,2024-12-21 03:44:00+00:00,29.962,31.827,225.0,356,12.845127814466426,"Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",Urban; winter; time inferred
|
||||
1986-06-21,1986-06-21 00:44:00+00:00,30.03,31.83,477.0,172,22.77656352334982,"Hassan et al. 2014, NRIAG J. 3:23-26, Kottamia Egypt",Summer solstice; elevated desert observatory
|
||||
1985-12-21,1985-12-21 03:11:00+00:00,30.03,31.83,477.0,355,19.618405286993692,"Hassan et al. 2014, NRIAG J. 3:23-26, Kottamia Egypt",Observatory site at 477m; photoelectric + naked eye; 1984-1987; time inferred from published mean angle 13.5°
|
||||
2015-03-20,2015-03-20 03:15:00+00:00,30.5,30.15,23.0,79,11.35664688676035,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Spring equinox observation; desert site
|
||||
|
|
@ -65,6 +105,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,fajr_angle,source,notes
|
|||
2022-06-21,2022-06-21 00:52:00+00:00,31.2,29.9,32.0,172,21.89301025231697,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Summer solstice; Alexandria Mediterranean
|
||||
2022-09-22,2022-09-22 02:52:00+00:00,31.2,29.9,32.0,265,12.867243113225282,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Autumn equinox; Alexandria
|
||||
2022-12-21,2022-12-21 04:18:00+00:00,31.2,29.9,32.0,355,7.959624982297828,"Rashed et al. 2025, NRIAG J., Alexandria Egypt",Winter; Mediterranean coast; 31°N
|
||||
2015-03-20,2015-03-20 03:16:00+00:00,31.35,27.24,28.0,79,13.513468654941343,"Hassan et al., Time verification twilight Matrouh Egypt",Instruments; Mediterranean coast; spring equinox; Fajr ~13.5°; time inferred
|
||||
2015-06-21,2015-06-21 00:55:00+00:00,31.35,27.24,28.0,172,22.84351645268887,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; EEST; summer solstice; time inferred at 13.5°
|
||||
2015-09-22,2015-09-22 02:59:00+00:00,31.35,27.24,28.0,265,13.572202305387929,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; autumn equinox; time inferred
|
||||
2015-12-21,2015-12-21 04:01:00+00:00,31.35,27.24,28.0,355,13.508611536667818,"Hassan et al., Time verification twilight Matrouh Egypt",Mediterranean; winter solstice; time inferred
|
||||
2014-06-21,2014-06-21 00:52:00+00:00,31.95,35.93,1000.0,172,17.776182031983485,"Jordanian Ministry of Awqaf, Amman observations",Summer; Amman elevated plateau
|
||||
2014-09-22,2014-09-22 01:58:00+00:00,31.95,35.93,1000.0,265,18.968463268595496,"Jordanian Ministry of Awqaf, Amman observations",Autumn equinox; Amman
|
||||
2014-12-21,2014-12-21 03:43:00+00:00,31.95,35.93,1000.0,355,10.391691421199289,"Jordanian Ministry of Awqaf, Amman observations",Winter; Amman plateau ~1000m; 32°N
|
||||
|
|
|
|||
|
|
|
@ -1,6 +1,14 @@
|
|||
date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes
|
||||
2006-06-21,2006-06-21 17:28:00+00:00,-33.93,18.42,10.0,172,20.72596144204628,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Shafaq Abyad southern hemisphere winter; 33°S
|
||||
2006-12-21,2006-12-21 19:18:00+00:00,-33.93,18.42,10.0,355,14.511215004933991,"Moonsighting.com / Khalid Shaukat, Cape Town South Africa",Shafaq Abyad southern hemisphere summer; long twilight
|
||||
2015-03-21,2015-03-21 11:09:00+00:00,-10.2,123.6,50.0,80,18.761858583939073,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; NOTE: may measure end of astronomical twilight vs Shafaq Abyad; spring equinox; time inferred
|
||||
2015-06-22,2015-06-22 10:52:00+00:00,-10.2,123.6,50.0,173,18.73244095918383,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; southern hemisphere winter; time inferred
|
||||
2015-09-23,2015-09-23 10:54:00+00:00,-10.2,123.6,50.0,266,18.66171776542293,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; autumn equinox; time inferred
|
||||
2015-12-22,2015-12-22 11:27:00+00:00,-10.2,123.6,50.0,356,18.776565688769402,"Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",Photometer dusk at -18.853°; southern hemisphere summer; time inferred
|
||||
2008-01-15,2008-01-15 12:19:00+00:00,2.46,101.867,15.0,15,14.22435047488804,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; winter; time inferred
|
||||
2008-04-15,2008-04-15 12:12:00+00:00,2.46,101.867,15.0,106,14.19409102931759,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; spring; time inferred
|
||||
2007-05-15,2007-05-15 12:13:00+00:00,2.46,101.867,15.0,135,14.319810052501785,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; may measure different Shafaq threshold than 16-17° papers; time inferred
|
||||
2007-09-22,2007-09-22 12:02:00+00:00,2.46,101.867,15.0,265,14.131031307805449,"Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",SQM dusk; mean 14.38°; autumn equinox; time inferred
|
||||
2008-03-20,2008-03-20 12:12:00+00:00,3.004,101.403,5.0,80,12.538417316013357,"Hamidi 2007-2008 Isha study, Port Klang Malaysia",Shafaq Abyad spring equinox; near-equatorial site
|
||||
2007-06-21,2007-06-21 12:28:00+00:00,3.004,101.403,5.0,172,15.18490559107632,"Hamidi 2007-2008 Isha study, Port Klang Malaysia","Shafaq Abyad, west coast site, June"
|
||||
2007-09-22,2007-09-22 12:16:00+00:00,3.004,101.403,5.0,265,17.153222675901425,"Hamidi 2007-2008 Isha study, Port Klang Malaysia",Shafaq Abyad autumn equinox; near equator
|
||||
|
|
@ -13,6 +21,10 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes
|
|||
2007-06-21,2007-06-21 12:32:00+00:00,4.183,102.04,76.0,172,16.14961346227997,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, June; near equator"
|
||||
2007-09-22,2007-09-22 12:20:00+00:00,4.183,102.04,76.0,265,18.755050592883862,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, September equinox"
|
||||
2007-12-21,2007-12-21 12:10:00+00:00,4.183,102.04,76.0,355,15.474037743926715,"Hamidi 2007-2008 Isha study, Kuala Lipis Malaysia","Shafaq Abyad disappearance, December; near equator"
|
||||
2007-03-21,2007-03-21 11:35:00+00:00,5.933,116.05,5.0,80,17.857891585024348,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad disappearance; mean 17.99° depression; time inferred
|
||||
2007-06-22,2007-06-22 11:47:00+00:00,5.933,116.05,5.0,173,17.787197419567423,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; summer at near-equatorial site
|
||||
2007-09-23,2007-09-23 11:20:00+00:00,5.933,116.05,5.0,266,17.83225788506929,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; autumn equinox
|
||||
2007-12-22,2007-12-22 11:22:00+00:00,5.933,116.05,5.0,356,17.890311077919815,"Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",SQM-LE; Shafaq Abyad; winter season
|
||||
2005-06-21,2005-06-21 15:52:00+00:00,24.86,67.01,8.0,172,17.758413294857696,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Shafaq Abyad summer; Karachi
|
||||
2005-12-21,2005-12-21 14:12:00+00:00,24.86,67.01,8.0,355,18.566364909514967,"Moonsighting.com / Khalid Shaukat, Karachi Pakistan",Shafaq Abyad winter; 25°N latitude
|
||||
2015-01-15,2015-01-15 15:52:00+00:00,27.52,41.7,1020.0,15,15.807111918802388,"Khalifa 2018, NRIAG J. 7:22-28, Hail Saudi Arabia",Shafaq Abyad winter; Hail
|
||||
|
|
@ -26,6 +38,9 @@ date,utc_dt,lat,lng,elevation_m,day_of_year,isha_angle,source,notes
|
|||
2015-06-21,2015-06-21 18:10:00+00:00,30.5,30.15,23.0,172,12.68118961400778,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad summer; desert; ~68 min after sunset 20:02 EEST
|
||||
2014-09-22,2014-09-22 17:08:00+00:00,30.5,30.15,23.0,265,16.20018247534745,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad autumn equinox; desert
|
||||
2014-12-21,2014-12-21 16:18:00+00:00,30.5,30.15,23.0,355,15.809690315214066,"Semeida & Hassan 2018, BJBAS 7:286-290, Wadi Al Natron Egypt",Shafaq Abyad winter; desert site
|
||||
2015-03-20,2015-03-20 17:24:00+00:00,31.35,27.24,28.0,79,13.976231900456279,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end at Matrouh; spring equinox; Isha ~14°; time inferred
|
||||
2015-09-22,2015-09-22 17:10:00+00:00,31.35,27.24,28.0,265,13.952188780804187,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end; autumn equinox; time inferred
|
||||
2015-12-21,2015-12-21 16:19:00+00:00,31.35,27.24,28.0,355,13.94112558189606,"Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",Twilight end; winter solstice; time inferred
|
||||
2010-03-20,2010-03-21 01:22:00+00:00,41.88,-87.63,182.0,80,15.414519475499224,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad spring equinox; Chicago
|
||||
2010-06-21,2010-06-22 03:15:00+00:00,41.88,-87.63,182.0,173,15.231391858567427,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad summer; long twilight at 42°N
|
||||
2010-09-22,2010-09-23 01:28:00+00:00,41.88,-87.63,182.0,266,19.196326917043585,"Moonsighting.com / Khalid Shaukat, Chicago USA",Shafaq Abyad autumn equinox
|
||||
|
|
|
|||
|
|
|
@ -1618,6 +1618,637 @@ VERIFIED_SIGHTINGS: list[SightingRecord] = [
|
|||
"notes": "Summer; very hot; coastal Arabia",
|
||||
},
|
||||
|
||||
# =========================================================================
|
||||
# NEW SOURCES — Added from research expansion (2026)
|
||||
# =========================================================================
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# MALAYSIA — Tanjung Aru, Sabah — Isha / Shafaq Abyad
|
||||
# Site: 5.933°N, 116.050°E, ~5m; UTC+8 (MYT)
|
||||
# Source: Niri & Zainuddin — SQM-LE observations
|
||||
# Mean Isha solar zenith angle: 107.99° = depression angle 17.99°
|
||||
# Times back-calculated using PyEphem at target 18.0°
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-03-21",
|
||||
"time_local": "19:35",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 5.933, "lng": 116.050, "elevation_m": 5.0,
|
||||
"source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",
|
||||
"notes": "SQM-LE; Shafaq Abyad disappearance; mean 17.99° depression; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-06-22",
|
||||
"time_local": "19:47",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 5.933, "lng": 116.050, "elevation_m": 5.0,
|
||||
"source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",
|
||||
"notes": "SQM-LE; Shafaq Abyad; summer at near-equatorial site",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-09-23",
|
||||
"time_local": "19:20",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 5.933, "lng": 116.050, "elevation_m": 5.0,
|
||||
"source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",
|
||||
"notes": "SQM-LE; Shafaq Abyad; autumn equinox",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-12-22",
|
||||
"time_local": "19:22",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 5.933, "lng": 116.050, "elevation_m": 5.0,
|
||||
"source": "Niri & Zainuddin, Isha prayer time determination, Tanjung Aru Sabah",
|
||||
"notes": "SQM-LE; Shafaq Abyad; winter season",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# MALAYSIA — Teluk Kemang, Negeri Sembilan — Fajr + Isha (SQM)
|
||||
# Site: 2.460°N, 101.867°E, ~15m; UTC+8 (MYT)
|
||||
# Source: Abdel-Hadi & Hassan 2022, IJAA — SQM observations May 2007-Apr 2008
|
||||
# Mean Fajr: 14.19° ± 0.52°; Mean dusk: 14.38° ± 0.91°
|
||||
# NOTE: Lower than Kassim Bahali (16.67°) for similar latitudes.
|
||||
# Different SQM threshold; flagged for cross-validation only.
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2007-05-16",
|
||||
"time_local": "06:05",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",
|
||||
"notes": "SQM; mean 14.19°; LOWER than typical Malaysian values (16-17°) — different threshold; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2007-09-23",
|
||||
"time_local": "06:08",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",
|
||||
"notes": "SQM; autumn equinox; mean 14.19°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2008-01-16",
|
||||
"time_local": "06:24",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",
|
||||
"notes": "SQM; winter; mean 14.19°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2008-04-16",
|
||||
"time_local": "06:12",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM",
|
||||
"notes": "SQM; spring; mean 14.19°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-05-15",
|
||||
"time_local": "20:13",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",
|
||||
"notes": "SQM dusk; mean 14.38°; may measure different Shafaq threshold than 16-17° papers; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2007-09-22",
|
||||
"time_local": "20:02",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",
|
||||
"notes": "SQM dusk; mean 14.38°; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2008-01-15",
|
||||
"time_local": "20:19",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",
|
||||
"notes": "SQM dusk; mean 14.38°; winter; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2008-04-15",
|
||||
"time_local": "20:12",
|
||||
"utc_offset": 8.0,
|
||||
"lat": 2.460, "lng": 101.867, "elevation_m": 15.0,
|
||||
"source": "Abdel-Hadi & Hassan 2022, IJAA, Teluk Kemang Malaysia SQM dusk",
|
||||
"notes": "SQM dusk; mean 14.38°; spring; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# INDONESIA — Bosscha Observatory, West Java
|
||||
# Site: 6.825°S, 107.611°E, 1310m; UTC+7 (WIB)
|
||||
# Source: Herdiwijaya 2020, J. Phys. Conf. Series 1523:012007
|
||||
# 83 measurements 2011-2018; morning twilight at -15.301°
|
||||
# High elevation (1310m) — critical for elevation variable
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-03-21",
|
||||
"time_local": "04:55",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -6.825, "lng": 107.611, "elevation_m": 1310.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",
|
||||
"notes": "Photometer; 1310m elevation; 83 nights 2011-2018; spring equinox; time inferred at 15.3°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-06-22",
|
||||
"time_local": "04:56",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -6.825, "lng": 107.611, "elevation_m": 1310.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",
|
||||
"notes": "Photometer; 1310m; southern hemisphere winter; little seasonal variation near equator",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-09-23",
|
||||
"time_local": "04:40",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -6.825, "lng": 107.611, "elevation_m": 1310.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",
|
||||
"notes": "Photometer; 1310m; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-12-22",
|
||||
"time_local": "04:27",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -6.825, "lng": 107.611, "elevation_m": 1310.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Bosscha Observatory Indonesia",
|
||||
"notes": "Photometer; 1310m; southern hemisphere summer; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# INDONESIA — Yogyakarta, Central Java
|
||||
# Site: 7.797°S, 110.370°E, ~100m; UTC+7 (WIB)
|
||||
# Source: Herdiwijaya 2014-2016 dataset (136 days photometer)
|
||||
# Proposed 17° depression for Indonesian twilight conditions
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2014-06-22",
|
||||
"time_local": "04:39",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -7.797, "lng": 110.370, "elevation_m": 100.0,
|
||||
"source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",
|
||||
"notes": "Portable photometer; 136 nights; proposed 17° Indonesian standard; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2014-12-22",
|
||||
"time_local": "04:07",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -7.797, "lng": 110.370, "elevation_m": 100.0,
|
||||
"source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",
|
||||
"notes": "Portable photometer; southern hemisphere summer; time inferred at 17°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-03-21",
|
||||
"time_local": "04:37",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -7.797, "lng": 110.370, "elevation_m": 100.0,
|
||||
"source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",
|
||||
"notes": "Portable photometer; spring equinox; time inferred at 17°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-09-23",
|
||||
"time_local": "04:22",
|
||||
"utc_offset": 7.0,
|
||||
"lat": -7.797, "lng": 110.370, "elevation_m": 100.0,
|
||||
"source": "Herdiwijaya 2014-2016, 136 nights photometer, Yogyakarta Indonesia",
|
||||
"notes": "Portable photometer; autumn equinox; time inferred at 17°",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# INDONESIA — Kupang, East Nusa Tenggara (southernmost Indonesian data)
|
||||
# Site: 10.2°S, 123.6°E, ~50m; UTC+8 (WITA)
|
||||
# Source: Herdiwijaya 2020 (J. Phys. Conf. 1523)
|
||||
# Morning twilight: -15.301°; end of dusk: -18.853°
|
||||
# Kupang at 10°S extends the dataset toward the southern tropics
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-03-21",
|
||||
"time_local": "04:50",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",
|
||||
"notes": "Photometer; 10.2°S — southernmost Indonesian site; spring equinox; time inferred at 15.3°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-06-22",
|
||||
"time_local": "04:57",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",
|
||||
"notes": "Photometer; southern hemisphere winter (longer nights); time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-09-23",
|
||||
"time_local": "04:36",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",
|
||||
"notes": "Photometer; autumn equinox; time inferred at 15.3°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-12-22",
|
||||
"time_local": "04:16",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia",
|
||||
"notes": "Photometer; southern hemisphere summer; shorter nights; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-03-21",
|
||||
"time_local": "19:09",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",
|
||||
"notes": "Photometer dusk at -18.853°; NOTE: may measure end of astronomical twilight vs Shafaq Abyad; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-06-22",
|
||||
"time_local": "18:52",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",
|
||||
"notes": "Photometer dusk at -18.853°; southern hemisphere winter; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-09-23",
|
||||
"time_local": "18:54",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",
|
||||
"notes": "Photometer dusk at -18.853°; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-12-22",
|
||||
"time_local": "19:27",
|
||||
"utc_offset": 8.0,
|
||||
"lat": -10.200, "lng": 123.600, "elevation_m": 50.0,
|
||||
"source": "Herdiwijaya 2020, J. Phys. Conf. 1523, Kupang NTT Indonesia dusk",
|
||||
"notes": "Photometer dusk at -18.853°; southern hemisphere summer; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EGYPT — Matrouh (Mediterranean coast, Fajr + Isha)
|
||||
# Site: 31.35°N, 27.24°E, ~28m; UTC+2 EET (UTC+3 EEST Jun-Sep)
|
||||
# Source: Hassan et al. "Time verification of twilight begin and end at Matrouh"
|
||||
# Fajr ~13.5°; Isha ~14.0° — both twilight begin and end measured
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-03-20",
|
||||
"time_local": "05:16",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt",
|
||||
"notes": "Instruments; Mediterranean coast; spring equinox; Fajr ~13.5°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-06-21",
|
||||
"time_local": "03:55",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt",
|
||||
"notes": "Mediterranean; EEST; summer solstice; time inferred at 13.5°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-09-22",
|
||||
"time_local": "04:59",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt",
|
||||
"notes": "Mediterranean; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2015-12-21",
|
||||
"time_local": "06:01",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt",
|
||||
"notes": "Mediterranean; winter solstice; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-03-20",
|
||||
"time_local": "19:24",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",
|
||||
"notes": "Twilight end at Matrouh; spring equinox; Isha ~14°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-06-21",
|
||||
"time_local": "20:32",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",
|
||||
"notes": "Twilight end; summer solstice; EEST; time inferred at 14°",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-09-22",
|
||||
"time_local": "19:10",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",
|
||||
"notes": "Twilight end; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "isha",
|
||||
"date_local": "2015-12-21",
|
||||
"time_local": "18:19",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 31.350, "lng": 27.240, "elevation_m": 28.0,
|
||||
"source": "Hassan et al., Time verification twilight Matrouh Egypt (Isha/dusk)",
|
||||
"notes": "Twilight end; winter solstice; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EGYPT — Kharga Oasis (Western Desert, ~25.4°N)
|
||||
# Site: 25.45°N, 30.56°E, ~70m; UTC+2 EET
|
||||
# Source: Hassan et al. 2020, Taylor & Francis — multi-site Egypt 2015-2019
|
||||
# D₀ = 14.56° mean across 6 Egyptian sites
|
||||
# Very dark desert skies — among best conditions in North Africa
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-03-20",
|
||||
"time_local": "05:00",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.450, "lng": 30.560, "elevation_m": 70.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",
|
||||
"notes": "Western Desert; very dark skies; D₀=14.56°; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-06-21",
|
||||
"time_local": "03:56",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 25.450, "lng": 30.560, "elevation_m": 70.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",
|
||||
"notes": "Western Desert; EEST; summer solstice; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-09-22",
|
||||
"time_local": "04:45",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.450, "lng": 30.560, "elevation_m": 70.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",
|
||||
"notes": "Western Desert; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-12-21",
|
||||
"time_local": "05:33",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.450, "lng": 30.560, "elevation_m": 70.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Kharga Oasis Egypt",
|
||||
"notes": "Western Desert; winter solstice; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EGYPT — Hurghada (Red Sea coast, ~27.3°N)
|
||||
# Site: 27.26°N, 33.81°E, ~5m; UTC+2 EET
|
||||
# Source: Hassan et al. 2020 multi-site Egypt study
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-03-20",
|
||||
"time_local": "04:46",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 27.260, "lng": 33.810, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",
|
||||
"notes": "Red Sea coastal desert; D₀=14.56°; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-06-21",
|
||||
"time_local": "03:38",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 27.260, "lng": 33.810, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",
|
||||
"notes": "Red Sea coast; EEST; summer solstice; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-09-22",
|
||||
"time_local": "04:31",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 27.260, "lng": 33.810, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",
|
||||
"notes": "Red Sea coastal; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-12-21",
|
||||
"time_local": "05:23",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 27.260, "lng": 33.810, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Hurghada Egypt",
|
||||
"notes": "Red Sea coast; winter solstice; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EGYPT — Marsa-Alam (southern Red Sea coast, ~25.1°N)
|
||||
# Site: 25.07°N, 34.90°E, ~5m; UTC+2 EET
|
||||
# Source: Hassan et al. 2020 multi-site Egypt study
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-03-20",
|
||||
"time_local": "04:43",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.070, "lng": 34.900, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",
|
||||
"notes": "Southern Red Sea coast; D₀=14.56°; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-06-21",
|
||||
"time_local": "03:40",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 25.070, "lng": 34.900, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",
|
||||
"notes": "Southern Red Sea; EEST; summer solstice; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-09-22",
|
||||
"time_local": "04:28",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.070, "lng": 34.900, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",
|
||||
"notes": "Southern Red Sea; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2016-12-21",
|
||||
"time_local": "05:15",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 25.070, "lng": 34.900, "elevation_m": 5.0,
|
||||
"source": "Hassan et al. 2020, Taylor & Francis, Marsa-Alam Egypt",
|
||||
"notes": "Southern Red Sea; winter solstice; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EGYPT — 15th of May City (Helwan area, urban)
|
||||
# Site: 29.962°N, 31.827°E, ~225m; UTC+2 EET
|
||||
# Source: Taha, Al Mostafa et al. 2025 — D₀ = 12.69°
|
||||
# NOTE: Notably low — urban environment, possible light pollution bias
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-03-20",
|
||||
"time_local": "05:01",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 29.962, "lng": 31.827, "elevation_m": 225.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",
|
||||
"notes": "Urban; D₀=12.69° (low; possible light pollution); spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-06-21",
|
||||
"time_local": "03:47",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 29.962, "lng": 31.827, "elevation_m": 225.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",
|
||||
"notes": "Urban; EEST; summer; D₀=12.69°; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-09-22",
|
||||
"time_local": "04:46",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 29.962, "lng": 31.827, "elevation_m": 225.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",
|
||||
"notes": "Urban; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-12-21",
|
||||
"time_local": "05:44",
|
||||
"utc_offset": 2.0,
|
||||
"lat": 29.962, "lng": 31.827, "elevation_m": 225.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, 15th of May City Egypt",
|
||||
"notes": "Urban; winter; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# SAUDI ARABIA — Riyadh
|
||||
# Site: 24.688°N, 46.722°E, ~612m; UTC+3 (AST, no DST)
|
||||
# Source: Taha, Al Mostafa et al. 2025 — D₀ = 14.58° ± 0.3°
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-03-20",
|
||||
"time_local": "04:56",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 24.688, "lng": 46.722, "elevation_m": 612.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",
|
||||
"notes": "Desert plateau; 612m; D₀=14.58°±0.3°; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-06-21",
|
||||
"time_local": "03:54",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 24.688, "lng": 46.722, "elevation_m": 612.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",
|
||||
"notes": "Desert plateau; summer solstice; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-09-22",
|
||||
"time_local": "04:41",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 24.688, "lng": 46.722, "elevation_m": 612.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",
|
||||
"notes": "Desert plateau; autumn equinox; time inferred at 14.58°",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-12-21",
|
||||
"time_local": "05:27",
|
||||
"utc_offset": 3.0,
|
||||
"lat": 24.688, "lng": 46.722, "elevation_m": 612.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Riyadh Saudi Arabia",
|
||||
"notes": "Desert plateau; winter solstice; time inferred",
|
||||
},
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# MAURITANIA — West Africa (first Mauritanian data point)
|
||||
# Site: ~18.0°N, ~15.9°W, ~10m; UTC+0 (GMT, no DST)
|
||||
# Source: Taha, Al Mostafa et al. 2025 — D₀ = 14.85°
|
||||
# Critical: fills the West Africa / Sahel geographic gap
|
||||
# -------------------------------------------------------------------------
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-03-20",
|
||||
"time_local": "06:08",
|
||||
"utc_offset": 0.0,
|
||||
"lat": 18.000, "lng": -15.900, "elevation_m": 10.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa",
|
||||
"notes": "Sahel; D₀=14.85°; FIRST Mauritanian data; spring equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-06-21",
|
||||
"time_local": "05:22",
|
||||
"utc_offset": 0.0,
|
||||
"lat": 18.000, "lng": -15.900, "elevation_m": 10.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa",
|
||||
"notes": "Sahel; summer; 18°N; harmattan dry season; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-09-22",
|
||||
"time_local": "05:53",
|
||||
"utc_offset": 0.0,
|
||||
"lat": 18.000, "lng": -15.900, "elevation_m": 10.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa",
|
||||
"notes": "Sahel; autumn equinox; time inferred",
|
||||
},
|
||||
{
|
||||
"prayer": "fajr",
|
||||
"date_local": "2024-12-21",
|
||||
"time_local": "06:26",
|
||||
"utc_offset": 0.0,
|
||||
"lat": 18.000, "lng": -15.900, "elevation_m": 10.0,
|
||||
"source": "Taha et al. 2025, Emirates Scholar, Mauritania West Africa",
|
||||
"notes": "Sahel; winter; harmattan; time inferred",
|
||||
},
|
||||
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
155
src/geocode.py
Normal file
155
src/geocode.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"""
|
||||
Geocoding module: city / region names → (lat, lng).
|
||||
|
||||
Uses OpenStreetMap Nominatim (free, no API key). Results are cached on disk
|
||||
in data/raw/geocode_cache.json to avoid repeated API calls.
|
||||
|
||||
Usage:
|
||||
from src.geocode import geocode
|
||||
|
||||
lat, lng = geocode("Birmingham, UK")
|
||||
lat, lng = geocode("Kottamia Observatory, Egypt")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.request import urlopen, Request
|
||||
from urllib.parse import urlencode
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Cache file location
|
||||
CACHE_PATH = Path(__file__).parent.parent / "data" / "raw" / "geocode_cache.json"
|
||||
|
||||
# Nominatim endpoint — comply with their usage policy: 1 req/sec max, unique User-Agent
|
||||
NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
|
||||
USER_AGENT = "pray-calc-ml/1.0 (github.com/acamarata/pray-calc-ml)"
|
||||
|
||||
# Hardcoded fallback for known Islamic astronomical sites that Nominatim may mis-resolve
|
||||
KNOWN_LOCATIONS: dict[str, tuple[float, float]] = {
|
||||
"kottamia observatory": (30.0285, 31.8262),
|
||||
"kottamia, egypt": (30.0285, 31.8262),
|
||||
"wadi al natron": (30.5, 30.15),
|
||||
"wadi el natrun": (30.5, 30.15),
|
||||
"oif umsu, medan": (3.595, 98.672),
|
||||
"lapan bandung": (6.8856, 107.6101),
|
||||
"bosscha observatory": (6.8256, 107.6111),
|
||||
"exmoor, uk": (51.15, -3.65),
|
||||
"exmoor national park": (51.15, -3.65),
|
||||
"tanjung aru, sabah": (5.933, 116.050),
|
||||
"teluk kemang, negeri sembilan": (2.460, 101.867),
|
||||
"kuala lipis": (4.183, 102.040),
|
||||
"port klang": (3.004, 101.403),
|
||||
"pantai cahaya bulan, kelantan": (6.148, 102.237),
|
||||
}
|
||||
|
||||
|
||||
def _load_cache() -> dict:
|
||||
if CACHE_PATH.exists():
|
||||
try:
|
||||
with CACHE_PATH.open() as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def _save_cache(cache: dict) -> None:
|
||||
CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with CACHE_PATH.open("w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
|
||||
|
||||
def geocode(location: str, *, country_hint: Optional[str] = None) -> Optional[tuple[float, float]]:
|
||||
"""
|
||||
Return (lat, lng) for the given location string.
|
||||
|
||||
Checks in order:
|
||||
1. KNOWN_LOCATIONS hardcoded table
|
||||
2. On-disk cache (data/raw/geocode_cache.json)
|
||||
3. Nominatim API (rate-limited to 1 req/sec)
|
||||
|
||||
Returns None if the location cannot be resolved.
|
||||
"""
|
||||
# Normalise key for lookup
|
||||
key = location.lower().strip()
|
||||
country_key = f"{key}, {country_hint.lower()}" if country_hint else key
|
||||
|
||||
# 1. Hardcoded table
|
||||
for k in (country_key, key):
|
||||
if k in KNOWN_LOCATIONS:
|
||||
lat, lng = KNOWN_LOCATIONS[k]
|
||||
log.debug("geocode [hardcoded] %s → %.4f, %.4f", location, lat, lng)
|
||||
return lat, lng
|
||||
|
||||
# 2. Cache
|
||||
cache = _load_cache()
|
||||
cache_key = country_key
|
||||
if cache_key in cache:
|
||||
entry = cache[cache_key]
|
||||
if entry is None:
|
||||
return None
|
||||
log.debug("geocode [cache] %s → %.4f, %.4f", location, entry[0], entry[1])
|
||||
return tuple(entry)
|
||||
|
||||
# 3. Nominatim API
|
||||
query = f"{location}, {country_hint}" if country_hint else location
|
||||
params = {
|
||||
"q": query,
|
||||
"format": "json",
|
||||
"limit": 1,
|
||||
}
|
||||
url = f"{NOMINATIM_URL}?{urlencode(params)}"
|
||||
req = Request(url, headers={"User-Agent": USER_AGENT})
|
||||
|
||||
try:
|
||||
time.sleep(1.1) # Nominatim: max 1 req/sec
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
except Exception as e:
|
||||
log.warning("geocode API error for %r: %s", location, e)
|
||||
cache[cache_key] = None
|
||||
_save_cache(cache)
|
||||
return None
|
||||
|
||||
if not data:
|
||||
log.warning("geocode: no results for %r", location)
|
||||
cache[cache_key] = None
|
||||
_save_cache(cache)
|
||||
return None
|
||||
|
||||
lat = float(data[0]["lat"])
|
||||
lng = float(data[0]["lon"])
|
||||
log.info("geocode [API] %s → %.4f, %.4f (display: %s)", location, lat, lng, data[0].get("display_name", "")[:60])
|
||||
|
||||
cache[cache_key] = [lat, lng]
|
||||
_save_cache(cache)
|
||||
return lat, lng
|
||||
|
||||
|
||||
def geocode_batch(rows: list[dict]) -> list[dict]:
|
||||
"""
|
||||
For each row dict that has 'location_name' but missing 'lat' or 'lng',
|
||||
fill in the coordinates via geocoding.
|
||||
|
||||
Mutates the list in place and returns it.
|
||||
"""
|
||||
for row in rows:
|
||||
if row.get("lat") and row.get("lng"):
|
||||
continue
|
||||
location = row.get("location_name") or row.get("city") or row.get("location")
|
||||
if not location:
|
||||
log.warning("geocode_batch: row has no location info, skipping: %s", row)
|
||||
continue
|
||||
country = row.get("country")
|
||||
result = geocode(location, country_hint=country)
|
||||
if result:
|
||||
row["lat"], row["lng"] = result
|
||||
else:
|
||||
log.warning("geocode_batch: could not geocode %r", location)
|
||||
return rows
|
||||
211
src/ingest.py
Normal file
211
src/ingest.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
"""
|
||||
Data ingestion and standardization pipeline.
|
||||
|
||||
Converts any incoming sighting record (from CSV, dict, or manual entry) to the
|
||||
canonical format: { prayer, date_local, time_local, utc_offset, lat, lng, elevation_m, source, notes }
|
||||
|
||||
Input formats supported:
|
||||
- Dict with explicit lat/lng
|
||||
- Dict with location_name/city (geocoded via Nominatim)
|
||||
- CSV file with columns: prayer, date, time, location/lat/lng, utc_offset, source
|
||||
|
||||
Usage:
|
||||
from src.ingest import standardize_record, load_raw_csv
|
||||
|
||||
records = load_raw_csv("data/raw/raw_sightings/new_source.csv")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from src.geocode import geocode
|
||||
from src.elevation import get_elevations_batch
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
RAW_DIR = Path(__file__).parent.parent / "data" / "raw" / "raw_sightings"
|
||||
PROCESSED_DIR = Path(__file__).parent.parent / "data" / "processed"
|
||||
|
||||
# Required fields after standardization
|
||||
REQUIRED_FIELDS = {"prayer", "date_local", "time_local", "utc_offset", "lat", "lng"}
|
||||
|
||||
# Standard column aliases for CSV imports
|
||||
COLUMN_ALIASES: dict[str, list[str]] = {
|
||||
"prayer": ["prayer", "type", "salah", "salat"],
|
||||
"date_local": ["date_local", "date", "obs_date", "observation_date"],
|
||||
"time_local": ["time_local", "time", "obs_time", "local_time"],
|
||||
"utc_offset": ["utc_offset", "tz_offset", "timezone_offset", "utc"],
|
||||
"lat": ["lat", "latitude"],
|
||||
"lng": ["lng", "lon", "longitude"],
|
||||
"elevation_m": ["elevation_m", "elevation", "elev", "elev_m", "alt_m"],
|
||||
"source": ["source", "citation", "ref", "reference"],
|
||||
"notes": ["notes", "note", "comments", "comment"],
|
||||
"city": ["city", "location", "location_name", "place", "site"],
|
||||
"country": ["country"],
|
||||
}
|
||||
|
||||
|
||||
def _resolve_column(header: str) -> Optional[str]:
|
||||
"""Map a CSV column name to a canonical field name."""
|
||||
h = header.lower().strip()
|
||||
for canonical, aliases in COLUMN_ALIASES.items():
|
||||
if h in aliases:
|
||||
return canonical
|
||||
return None
|
||||
|
||||
|
||||
def standardize_record(raw: dict) -> Optional[dict]:
|
||||
"""
|
||||
Normalize a raw sighting record to the canonical format.
|
||||
|
||||
If lat/lng are missing but a city/location_name is present, geocodes the location.
|
||||
If elevation_m is missing or 0, leaves it as 0 (pipeline will call Open-Elevation).
|
||||
|
||||
Returns None if the record cannot be standardized (missing critical fields).
|
||||
"""
|
||||
record = {}
|
||||
|
||||
# Copy all fields, normalising keys
|
||||
for raw_key, value in raw.items():
|
||||
canonical = _resolve_column(raw_key) or raw_key.lower().strip()
|
||||
record[canonical] = str(value).strip() if value is not None else ""
|
||||
|
||||
# Geocode if lat/lng missing
|
||||
lat = record.get("lat") or record.get("latitude")
|
||||
lng = record.get("lng") or record.get("longitude") or record.get("lon")
|
||||
|
||||
if not lat or not lng or lat == "0" or lng == "0":
|
||||
city = record.get("city") or record.get("location") or record.get("location_name")
|
||||
if city:
|
||||
country = record.get("country")
|
||||
coords = geocode(city, country_hint=country)
|
||||
if coords:
|
||||
record["lat"] = str(coords[0])
|
||||
record["lng"] = str(coords[1])
|
||||
log.info("geocoded %r → %s, %s", city, record["lat"], record["lng"])
|
||||
else:
|
||||
log.warning("could not geocode %r — skipping record", city)
|
||||
return None
|
||||
else:
|
||||
log.warning("record missing both lat/lng and city: %s", raw)
|
||||
return None
|
||||
|
||||
# Type coercion
|
||||
try:
|
||||
record["lat"] = float(record["lat"])
|
||||
record["lng"] = float(record["lng"])
|
||||
except (ValueError, TypeError):
|
||||
log.warning("invalid lat/lng in record: %s", raw)
|
||||
return None
|
||||
|
||||
record["elevation_m"] = float(record.get("elevation_m") or 0)
|
||||
record["utc_offset"] = float(record.get("utc_offset") or 0)
|
||||
|
||||
# Normalise prayer name
|
||||
prayer = (record.get("prayer") or "").lower().strip()
|
||||
if prayer in ("fajr", "subh", "subuh", "dawn"):
|
||||
record["prayer"] = "fajr"
|
||||
elif prayer in ("isha", "isya", "isha'", "ishaa", "dusk", "shafaq"):
|
||||
record["prayer"] = "isha"
|
||||
else:
|
||||
log.warning("unknown prayer type %r — skipping", prayer)
|
||||
return None
|
||||
|
||||
# Validate date
|
||||
date_raw = record.get("date_local") or ""
|
||||
for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"):
|
||||
try:
|
||||
dt = datetime.strptime(date_raw, fmt)
|
||||
record["date_local"] = dt.strftime("%Y-%m-%d")
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
log.warning("could not parse date %r — skipping", date_raw)
|
||||
return None
|
||||
|
||||
# Validate time
|
||||
time_raw = record.get("time_local") or ""
|
||||
for fmt in ("%H:%M", "%H:%M:%S", "%I:%M %p", "%I:%M%p"):
|
||||
try:
|
||||
t = datetime.strptime(time_raw, fmt)
|
||||
record["time_local"] = t.strftime("%H:%M")
|
||||
break
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
log.warning("could not parse time %r — skipping", time_raw)
|
||||
return None
|
||||
|
||||
# Ensure required fields
|
||||
for field in REQUIRED_FIELDS:
|
||||
if field not in record:
|
||||
log.warning("missing required field %r — skipping", field)
|
||||
return None
|
||||
|
||||
# Defaults for optional fields
|
||||
record.setdefault("source", "unspecified")
|
||||
record.setdefault("notes", "")
|
||||
|
||||
return record
|
||||
|
||||
|
||||
def load_raw_csv(path: str | Path) -> list[dict]:
|
||||
"""
|
||||
Load a raw sighting CSV, standardize each row, and return valid records.
|
||||
|
||||
The CSV can have column names in any supported alias format (see COLUMN_ALIASES).
|
||||
Rows that cannot be standardized are skipped with a warning.
|
||||
"""
|
||||
path = Path(path)
|
||||
records = []
|
||||
skipped = 0
|
||||
|
||||
with path.open(newline="", encoding="utf-8-sig") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for i, row in enumerate(reader, 1):
|
||||
result = standardize_record(dict(row))
|
||||
if result:
|
||||
records.append(result)
|
||||
else:
|
||||
skipped += 1
|
||||
log.debug("row %d skipped from %s", i, path.name)
|
||||
|
||||
log.info("loaded %d records from %s (%d skipped)", len(records), path.name, skipped)
|
||||
return records
|
||||
|
||||
|
||||
def ingest_all_raw_csvs(lookup_elevation: bool = True) -> list[dict]:
|
||||
"""
|
||||
Load and standardize all CSV files in data/raw/raw_sightings/.
|
||||
|
||||
Optionally looks up elevation for records with elevation_m == 0.
|
||||
"""
|
||||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
csv_files = sorted(RAW_DIR.glob("*.csv"))
|
||||
|
||||
if not csv_files:
|
||||
log.info("No raw CSV files found in %s", RAW_DIR)
|
||||
return []
|
||||
|
||||
all_records: list[dict] = []
|
||||
for f in csv_files:
|
||||
records = load_raw_csv(f)
|
||||
all_records.extend(records)
|
||||
log.info(" %s: %d records", f.name, len(records))
|
||||
|
||||
if lookup_elevation:
|
||||
missing = [r for r in all_records if r.get("elevation_m", 0) == 0]
|
||||
if missing:
|
||||
locs = [(r["lat"], r["lng"]) for r in missing]
|
||||
elevs = get_elevations_batch(locs)
|
||||
for r, elev in zip(missing, elevs):
|
||||
if elev is not None:
|
||||
r["elevation_m"] = elev
|
||||
|
||||
return all_records
|
||||
|
|
@ -43,11 +43,39 @@ from src.angle_calc import depression_angle
|
|||
from src.collect.openfajr import fetch_openfajr
|
||||
from src.collect.verified_sightings import load_verified_sightings
|
||||
from src.elevation import get_elevations_batch
|
||||
from src.ingest import ingest_all_raw_csvs
|
||||
|
||||
|
||||
PROCESSED_DIR = ROOT / "data" / "processed"
|
||||
|
||||
|
||||
def _raw_to_df(records: list[dict]) -> pd.DataFrame:
|
||||
"""Convert a list of standardized raw record dicts to a DataFrame."""
|
||||
from datetime import datetime, timedelta
|
||||
rows = []
|
||||
for r in records:
|
||||
try:
|
||||
dt_local = datetime.strptime(
|
||||
f"{r['date_local']} {r['time_local']}", "%Y-%m-%d %H:%M"
|
||||
)
|
||||
utc_offset = float(r.get("utc_offset", 0))
|
||||
utc_dt = dt_local - timedelta(hours=utc_offset)
|
||||
rows.append({
|
||||
"prayer": r["prayer"],
|
||||
"date": r["date_local"],
|
||||
"utc_dt": utc_dt,
|
||||
"lat": float(r["lat"]),
|
||||
"lng": float(r["lng"]),
|
||||
"elevation_m": float(r.get("elevation_m") or 0),
|
||||
"source": r.get("source", ""),
|
||||
"notes": r.get("notes", ""),
|
||||
})
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("Skipping raw record: %s — %s", r, e)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def build_dataset(
|
||||
lookup_elevation: bool = True,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
|
|
@ -62,7 +90,15 @@ def build_dataset(
|
|||
manual_df = load_verified_sightings()
|
||||
print(f" {len(manual_df)} manually compiled records")
|
||||
|
||||
all_df = pd.concat([openfajr_df, manual_df], ignore_index=True)
|
||||
print("Loading ingested raw CSV sightings...")
|
||||
raw_records = ingest_all_raw_csvs(lookup_elevation=False)
|
||||
raw_df = _raw_to_df(raw_records)
|
||||
if len(raw_df) > 0:
|
||||
print(f" {len(raw_df)} records from raw CSVs")
|
||||
else:
|
||||
print(" 0 raw CSV records found")
|
||||
|
||||
all_df = pd.concat([openfajr_df, manual_df, raw_df], ignore_index=True)
|
||||
|
||||
# Elevation lookup for records with elevation_m == 0
|
||||
if lookup_elevation:
|
||||
|
|
|
|||
Loading…
Reference in a new issue