pyCreateTh.py

This commit is contained in:
Alex38Lyon
2025-06-23 07:48:12 +02:00
parent 19d516997d
commit 0f2a5c9d73
14 changed files with 162 additions and 249 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+5 -4
View File
@@ -34,17 +34,18 @@ stationNamesInTh2 = -1
#################################################################################################
thFileDat = """
encoding utf-8
# File generated by pyCreateTh.py version {VERSION} date: {DATE}
# File generated by pyCreateTh.py version: {VERSION} date: {DATE}
survey {SURVEY_NAME} -title "{COMMENT}"
survey {SURVEY_NAME} -title "{SURVEY_TITLE}"
\t# {COMMENT}
\tcenterline
\t\tdate {SURVEY_DATE}
\t\t# team "{SURVEY_TEAM}"
{FIX_POINTS}
\t\t# explo-date "{EXPLO_DATE}"
\t\texplo-date {EXPLO_DATE}
\t\t# explo-team "{EXPLO_TEAM}"
{CORRECTIONS}{DECLINATION}
\t\tunits {LENGTH}
\t\tunits {COMPASS}
\t\tunits {CLINO}
+5 -68
View File
@@ -88,74 +88,11 @@ def compile_template(template, template_args, totReadMeError = "", **kwargs ):
#################################################################################################
# Compilation Therion (version sans blocage) #
#################################################################################################
def compile_fileOLd(filename, **kwargs):
tmpdir = os.path.dirname(filename)
log_file = join(tmpdir, "therion.log").replace("\\", "/")
therion_path = kwargs.get("therion_path", "therion")
timeout = kwargs.get("timeout", 60) # seconds
log.info(f"Start therion compilation file: {Colors.ENDC}{safe_relpath(filename)}")
try:
# Lancement du processus Therion
process = subprocess.Popen(
[therion_path, filename, "-l", log_file],
cwd=tmpdir,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
bufsize=1
)
# Fonction de lecture en temps réel (dans un thread séparé)
def read_output(proc):
try:
for line in proc.stdout:
line = line.rstrip()
lower_line = line.lower()
if "average loop error" in lower_line:
log.warning(f"[Therion_Compile] {Colors.ENDC}{line}")
elif "error" in lower_line:
log.error(f"[Therion_Compile] {Colors.ENDC}{line}")
elif "warning" in lower_line:
log.warning(f"[Therion_Compile] {Colors.ENDC}{line}")
else:
log.debug(f"[Therion_Compile] {Colors.ENDC}{line}")
except Exception as e:
log.warning(f"Reading Therion output: {Colors.ENDC}{e}")
# Démarrage du thread de lecture
output_thread = threading.Thread(target=read_output, args=(process,))
output_thread.start()
# Attente avec timeout
output_thread.join(timeout)
if output_thread.is_alive():
log.error(f"Therion compilation timed out after {Colors.ENDC}{timeout}{Colors.ERROR} seconds. Killing process...")
global_data.error_count += 1
process.kill()
output_thread.join()
process.wait()
# Vérification du code de retour
if process.returncode != 0:
log.error(f"Therion returned error code {Colors.ENDC}{process.returncode}")
global_data.error_count += 1
else:
log.info(f"Therion file: {Colors.ENDC}{safe_relpath(filename)}{Colors.GREEN} compilation succeeded")
except Exception as e:
log.error(f"Therion file: {Colors.ENDC}{safe_relpath(filename)}{Colors.ERROR} compilation error: {Colors.ENDC}{e}")
global_data.error_count += 1
def compile_file(filename, **kwargs):
tmpdir = os.path.dirname(filename)
log_file = join(tmpdir, "therion.log").replace("\\", "/")
therion_path = kwargs.get("therion_path", "therion")
timeout = kwargs.get("timeout", 60)
timeout = kwargs.get("timeout", 240)
log.info(f"Start therion compilation file: {Colors.ENDC}{safe_relpath(filename)}")
@@ -176,13 +113,13 @@ def compile_file(filename, **kwargs):
line = line.rstrip()
lower_line = line.lower()
if "average loop error" in lower_line:
log.warning(f"[Therion_Compile] {Colors.ENDC}{line}")
log.warning(f"[Therion Compile {os.path.basename(filename)[:-9]}] {Colors.ENDC}{line}")
elif "error" in lower_line:
log.error(f"[Therion_Compile] {Colors.ENDC}{line}")
log.error(f"[Therion_Compile {os.path.basename(filename)[:-9]}] {Colors.ENDC}{line}")
elif "warning" in lower_line:
log.warning(f"[Therion_Compile] {Colors.ENDC}{line}")
log.warning(f"[Therion Compile {os.path.basename(filename)[:-9]}] {Colors.ENDC}{line}")
else:
log.debug(f"[Therion_Compile] {Colors.ENDC}{line}")
log.debug(f"[Therion Compile {os.path.basename(filename)[:-9]}] {Colors.ENDC}{line}")
except Exception as e:
log.warning(f"Reading Therion output: {Colors.ENDC}{e}")
+139 -48
View File
@@ -16,21 +16,24 @@
Création Alex le 2025 06 09 :
Version 2025 06 16 : Création fonction create_th_folders
Ajout des fonctions pour mettre en log
Création de la fonction mak_to_th_file
Ajout des fonctions pour mak et dat
A venir :
En cours :
- gérer les visées orphelines dans une même survey
- gérer les updates (th, dat, mak)
- créer fonction pour faire habillage des th2 files, les jointures...
- reprendre l'option shot lines dans les th2 files pour supprimer les splays.
- créer l'option wall shot lines dans les th2 files.
- reprendre les options en ligne de commande, tester
- ajouter les commentaires et les déclinaisons dans les th files
- ajouter message pour les corrections non implantés
- trouver une solution pour les teams et les clubs manquants
- gérer le cas ou il y a 2 SurveyTitle identiques
- alléger les equates --> 1 fois dans le projet
- tester la nouvelle version de DAT (CORRECTION2 et suivants)
- tester différents cas et versions de dat et mak
- pb de equate avec points non valides (non répétable ?)
- vérifier le fonctionnement de merge survey (les valeurs compilées et les critères de compilation... voir )
- comparer résultats Therion - Compass (Stat, kml, etc....)
"""
@@ -506,7 +509,7 @@ def create_th_folders(ENTRY_FILE,
'configPath' : CONFIG_PATH,
'totData' : totdata,
'other_scraps_plan' : "",
'file_info' : f'# File generated by pyCreateTh.py (version {Version}) date: {datetime.now().strftime("%Y.%m.%d %H:%M:%S")}',
'file_info' : f'# File generated by pyCreateTh.py version: {Version} date: {datetime.now().strftime("%Y.%m.%d %H:%M:%S")}',
}
update_template_files(DEST_PATH + '/template.thconfig', config_vars, DEST_PATH + '/' + TH_NAME + '.thconfig')
@@ -899,7 +902,15 @@ def mak_to_th_file(ENTRY_FILE) :
totMapsExtended = ""
with alive_bar(QtySections, title=f"{Colors.GREEN}Surveys progress: {Colors.BLUE}", length = 20, enrich_print=False) as bar:
with alive_bar(QtySections,
title=f"{Colors.GREEN}Surveys progress: {Colors.BLUE}",
length = 20,
enrich_print=False,
stats=True, # Désactive les stats par défaut pour plus de lisibilité
elapsed=True, # Optionnel : masque le temps écoulé
monitor=True, # Optionnel : masque les métriques (ex: "eta")
bar="smooth" # Style de la barre (autres options: "smooth", "classic", "blocks")
) as bar:
with redirect_stdout(sys.__stdout__):
for file in datFiles:
@@ -977,8 +988,9 @@ def mak_to_th_file(ENTRY_FILE) :
# Pour chaque paire unique (i < j), on écrit la ligne 'equate'
for i in range(len(surveys)):
for j in range(i + 1, len(surveys)):
totdata +=f"\tequate {station}@{surveys[i]} {station}@{surveys[j]}\n"
# print(f"\tequate {station}@{surveys[i]} {station}@{surveys[j]}")
if surveys[i].split('.')[2] != surveys[j].split('.')[2]:
totdata +=f"\tequate {station}@{surveys[i]} {station}@{surveys[j]}\n"
# print(f"\tequate {station}@{surveys[i]} {station}@{surveys[j]}")
else:
log.info(f"No 'equats' found in {Colors.ENDC}{args.survey_file}")
@@ -1004,7 +1016,7 @@ def mak_to_th_file(ENTRY_FILE) :
'readMeList' : totReadMeList,
'errorList' : totReadMeError,
'fixPointList' : totReadMeFixPoint,
'file_info' : f"# File generated by pyCreateTh.py version {Version} date: {datetime.now().strftime("%Y.%m.%d-%H:%M:%S")}",
'file_info' : f"# File generated by pyCreateTh.py version: {Version} date: {datetime.now().strftime("%Y.%m.%d-%H:%M:%S")}",
}
DEST_PATH = os.path.dirname(args.survey_file) + '/' + SurveyTitleMak
@@ -1422,6 +1434,9 @@ def merge_duplicate_surveys(data, duplicates, id_offset=10000):
'DECLINATION': data[ids[0]]['DECLINATION'],
'FORMAT': data[ids[0]]['FORMAT'],
'CORRECTIONS': data[ids[0]]['CORRECTIONS'],
"CORRECTIONS2": data[ids[0]]['CORRECTIONS2'],
"DISCOVERY": data[ids[0]]['DISCOVERY'],
"PREFIX": data[ids[0]]['PREFIX'],
'DATA': [],
'STATION': [],
'SOURCE': []
@@ -1751,11 +1766,23 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
'DECLINATION': None,
'FORMAT': None,
'CORRECTIONS' : None,
"CORRECTIONS2": None,
"DISCOVERY": None,
"PREFIX": None,
'DATA' : [],
'STATION': [],
'SOURCE' : []
}
regex_patterns = {
"DECLINATION": r"DECLINATION:\s*([\d\.\-]+)",
"FORMAT": r"FORMAT:\s*([A-Za-z]+)",
"CORRECTIONS": r"CORRECTIONS:\s*([\d\.\-]+\s+[\d\.\-]+\s+[\d\.\-]+)",
"CORRECTIONS2": r"CORRECTIONS2:\s*([\d\.\-]+\s+[\d\.\-]+)",
"DISCOVERY": r"DISCOVERY:\s*(\d+\s+\d+\s+\d+)",
"PREFIX": r"PREFIX:\s*(\S+)"
}
# Parcourir les lignes de la section
lines = section.split('\n')
@@ -1788,7 +1815,7 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
date_parts = line.split(':', 1)[1].strip().split('COMMENT:', 1)
date = date_parts[0].strip()
jour, mois, annee = date.split()
date_convertie = f"{annee} {mois} {jour}"
date_convertie = f"{int(annee):04d} {int(mois):02d} {int(jour):02d}"
section_data['SURVEY_DATE'] = date_convertie
if section_data['SURVEY_DATE'] == None or section_data['SURVEY_DATE'] == '' :
section_data['SURVEY_DATE'] = "2000 01 01"
@@ -1801,23 +1828,13 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
elif NextLineSurveyTeam == True :
NextLineSurveyTeam = False
section_data['SURVEY_TEAM'] = line.strip()
elif line.startswith('DECLINATION:'):
# current_field = 'DECLINATION'
# Découper la ligne en trois parties
declination_part = line.split(':', 1)[1].strip()
elif line.startswith('DECLINATION:'):
for champ, pattern in regex_patterns.items():
match = re.search(pattern, line)
if match:
section_data[champ] = match.group(1).strip()
jumpLine = True # Sauter une ligne après la ligne DECLINATION
# Extraire DECLINATION (premier nombre)
declination_val = declination_part.split()[0]
section_data['DECLINATION'] = declination_val
# Trouver FORMAT et CORRECTIONS
if 'FORMAT:' in declination_part:
format_part = declination_part.split('FORMAT:', 1)[1]
format_val = format_part.split('CORRECTIONS:', 1)[0].strip()
section_data['FORMAT'] = format_val
if 'CORRECTIONS:' in format_part:
corrections_val = format_part.split('CORRECTIONS:', 1)[1].strip()
section_data['CORRECTIONS'] = corrections_val
else :
if line.strip() != '' :
@@ -1837,16 +1854,16 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
# Détecter les surveys avec plusieurs points de départ #
#################################################################################################
points = points_uniques(section_data, crs_wkt)
# points = points_uniques(section_data, crs_wkt)
if len(points) > 1 :
log.warning(f"Points {Colors.ENDC}{points}{Colors.WARNING} uniques dans la section {Colors.ENDC}{section_data['SURVEY_NAME']}")
# globalData.error_count += 1
# if len(points) > 1 :
# log.warning(f"Points {Colors.ENDC}{points}{Colors.WARNING} uniques dans la section {Colors.ENDC}{section_data['SURVEY_NAME']}")
# # globalData.error_count += 1
else :
log.debug(f"Points {Colors.ENDC}{points}{Colors.DEBUG} uniques dans la section {section_data['SURVEY_NAME']}")
# else :
# log.debug(f"Points {Colors.ENDC}{points}{Colors.DEBUG} uniques dans la section {section_data['SURVEY_NAME']}")
#################################################################################################
# Grouper les sections ayant même date team et un point commun #
#################################################################################################
@@ -1858,9 +1875,11 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
val2 = val1 - len(data)
bar(val2)
log.info(f"Read dat file: {Colors.ENDC}{shortCurentFile}{Colors.GREEN} with {Colors.ENDC}{len(data)}/{len(data)}{Colors.GREEN} survey")
if val2 != 0 :
log.info(f"Read dat file: {Colors.ENDC}{shortCurentFile}{Colors.INFO} with {Colors.ENDC}{len(data)}{Colors.GREEN}{Colors.INFO} survey(s) and merged {Colors.ENDC}{val2}")
bar(val2)
else :
log.info(f"Read dat file: {Colors.ENDC}{shortCurentFile}{Colors.INFO} with {Colors.ENDC}{len(data)}{Colors.INFO} survey(s)")
#################################################################################################
@@ -1904,16 +1923,47 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
output_file = f"{folderDest}\\Data\\{currentSurveyName}.th"
#################################################################################################
# gestion des CORRECTIONS #
#################################################################################################
_CorrectionValues = [float(val) for val in _line['CORRECTIONS'].strip().split()]
if all(val == 0.0 for val in _CorrectionValues) :
_corrections = ""
else :
_corrections = f"\t\t# Corrections: {_CorrectionValues[0]} {_CorrectionValues[1]} {_CorrectionValues[2]}, not yet implemented\n"
log.error(f"Corrections: {Colors.ENDC}{_CorrectionValues[0]} {_CorrectionValues[1]} {_CorrectionValues[2]}{Colors.ERROR}, not yet implemented in {Colors.ENDC}{currentSurveyName}")
totReadMeError += f"\tCorrections: {_CorrectionValues[0]} {_CorrectionValues[1]} {_CorrectionValues[2]}, not yet implemented in {currentSurveyName}\n"
globalData.error_count += 1
if _line['CORRECTIONS2'] != None :
_CorrectionValues3 = [float(val) for val in _line['CORRECTIONS2'].strip().split()]
if all(val == 0.0 for val in _CorrectionValues) :
_CorrectionValues3 = ""
else :
log.error(f"Corrections2: {Colors.ENDC}{_CorrectionValues[0]} {_CorrectionValues[1]} {_CorrectionValues[2]}{Colors.ERROR}, not yet implemented in {Colors.ENDC}{currentSurveyName}")
totReadMeError += f"\tCorrections2: {_CorrectionValues[0]} {_CorrectionValues[1]} {_CorrectionValues[2]}, not yet implemented in {currentSurveyName}\n"
globalData.error_count += 1
if _line['DISCOVERY'] != None :
date = _line['DISCOVERY'].strip()
jour, mois, annee = date.split()
discovery = f"{int(annee):04d} {int(mois):02d} {int(jour):02d}"
else :
discovery = f"{_line['SURVEY_DATE']} # '????'"
if _line['PREFIX'] != None :
log.error(f"PREFIX: {Colors.ENDC}{_line['PREFIX']}, not yet implemented in {Colors.ENDC}{currentSurveyName}")
totReadMeError += f"\tPREFIX: {_line['PREFIX']}, not yet implemented in {currentSurveyName}\n"
globalData.error_count += 1
SurveyNameCount = {
'surveyCount' :f"{currentSurveyName}",
'SURVEY_NAME': _line['SURVEY_NAME']
}
SurveyListEqui.append(SurveyNameCount)
#################################################################################################
# gestion des déclinaisons #
#################################################################################################
#################################################################################################
@@ -1955,12 +2005,22 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
dataFormat, length, compass, clino, totReadMeErrorDat = dat_survey_format_extract(_line, currentSurveyName, shortCurentFile, totReadMeErrorDat)
if "grads" in compass:
_compass = "grads"
else:
_compass = "degree"
#################################################################################################
# Gestion des formats
#################################################################################################
with open(str(output_file), "w+", encoding="utf-8") as f:
f.write(globalData.thFileDat.format(
VERSION = Version,
DATE=datetime.now().strftime("%Y.%m.%d-%H:%M:%S"),
# SURVEY_NAME = sanitize_filename(_line['SURVEY_NAME']),
SURVEY_NAME = f"{currentSurveyName}",
SURVEY_TITLE = _line['SURVEY_NAME'].replace("_", " "),
SURVEY_DATE = _line['SURVEY_DATE'],
SURVEY_TEAM = _line['SURVEY_TEAM'],
FORMAT = _line['FORMAT'],
@@ -1968,12 +2028,13 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
LENGTH = length,
CLINO = clino,
DATA_FORMAT = dataFormat,
CORRECTIONS = _line['CORRECTIONS'],
CORRECTIONS =_corrections,
DECLINATION = f"\t\tdeclination {_line['DECLINATION']} {_compass}\n" if (crs_wkt == "" and _line['DECLINATION'] != 0.0) else "",
DATA = formated_station_list(dfDATA, dataFormat, length, shortCurentFile),
COMMENT = sanitize_filename(_line['SURVEY_NAME'] + " " + _line['COMMENT']).replace('"', "'").replace('_', " "),
FIX_POINTS = fixPoint,
EXPLO_DATE = "????",
EXPLO_TEAM = "????",
EXPLO_DATE = discovery,
EXPLO_TEAM = f"{_line['SURVEY_TEAM']} # '????'",
SOURCE = '\n'.join('# ' + line for line in _line['SOURCE'].splitlines()),
)
)
@@ -2086,7 +2147,7 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
'thanksto' : globalData.thanksto,
'datat' : globalData.datat,
'wpage' : globalData.wpage,
'cs' : crs_wkt,
'cs' : crs_wkt if crs_wkt != "" else globalData.cs,
'totData' : totdata,
'configPath' : CONFIG_PATH,
'other_scraps_plan' : totMapsPlan,
@@ -2094,7 +2155,7 @@ def dat_to_th_files (ENTRY_FILE, fixPoints = [], crs_wkt = "", CONFIG_PATH = "",
'errorList' : totReadMeErrorDat,
'fixPointList' : totReadMeFixPoint,
'other_scraps_extended' : totMapsExtended,
'file_info' : f"# File generated by pyCreateTh.py version {Version} date: {datetime.now().strftime("%Y.%m.%d-%H:%M:%S")}",
'file_info' : f"# File generated by pyCreateTh.py version: {Version} date: {datetime.now().strftime("%Y.%m.%d-%H:%M:%S")}",
}
DEST_PATH = os.path.dirname(ENTRY_FILE) + '/' + SurveyTitle
@@ -2211,7 +2272,10 @@ if __name__ == u'__main__':
except ValueError as e:
log.critical(f"Reading config.ini file error: {Colors.ENDC}{e}")
exit(0)
#################################################################################################
# Fichier TH #
#################################################################################################
if args.survey_file[-2:].lower() == "th" :
flagErrorCompile, stat, totReadMeError, thread2 = create_th_folders(
ENTRY_FILE = abspath(args.survey_file),
@@ -2223,10 +2287,26 @@ if __name__ == u'__main__':
threads += thread2
fileTitle = sanitize_filename(os.path.basename(args.survey_file))[:-3]
#################################################################################################
# Fichier MAK #
#################################################################################################
elif args.survey_file[-3:].lower() == "mak" :
SurveyTitleMak = sanitize_filename(os.path.basename(abspath(args.survey_file))[:-4])
DEST_PATH = os.path.dirname(args.survey_file) + '/' + SurveyTitleMak
if os.path.isdir(DEST_PATH):
log.critical(f"The folder {Colors.ENDC}{SurveyTitleMak}{Colors.ERROR}{Colors.BOLD}, all ready exist : update mode is not possible for mak files")
exit(0)
fileTitle, thread2 = mak_to_th_file(abspath(args.survey_file))
threads += thread2
#################################################################################################
# Fichier DAT #
#################################################################################################
elif args.survey_file[-3:].lower() == "dat" :
_ConfigPath = "./"
@@ -2237,8 +2317,19 @@ if __name__ == u'__main__':
content, val = load_text_file_utf8(ABS_file, os.path.basename(ABS_file))
section = content.split('\x0c')
QtySections += len(section)
lines = section[0].split('\n')
if lines[0] !="" :
SurveyTitleDat = sanitize_filename(lines[0])
folderDest = os.path.dirname(args.survey_file) + "\\" + SurveyTitleDat
else :
SurveyTitleDat = sanitize_filename(os.path.basename(args.survey_file)[:-4])
folderDest = os.path.dirname(args.survey_file) + "\\" + SurveyTitleDat
if os.path.isdir(folderDest):
log.critical(f"The folder {Colors.ENDC}{SurveyTitleDat}{Colors.ERROR}{Colors.BOLD}, all ready exist : update mode is not possible for mak files")
exit(0)
with alive_bar(QtySections, title=f"{Colors.GREEN}Surveys progress: {Colors.BLUE}", length = 20, enrich_print=False) as bar:
with redirect_stdout(sys.__stdout__):