diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e6abdc..4788dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added choice 'blank' for qos in remap_restart python package +- Added hydrostatic question in remap_restart python package +- Added function to remap from stretched restarts to stretched restarts + ### Changed ### Fixed diff --git a/post/stats.F90 b/post/stats.F90 index 70d9de3..7232faa 100644 --- a/post/stats.F90 +++ b/post/stats.F90 @@ -1053,8 +1053,16 @@ end subroutine init_levs zlev(lev).eq.750 .or. zlev(lev).eq.600 .or. & zlev(lev).eq.500 .or. zlev(lev).eq.400 .or. & zlev(lev).eq.300 .or. zlev(lev).eq.200 .or. & - zlev(lev).eq.100 .or. zlev(lev).eq.70 .or. & - zlev(lev).eq.30 .or. zlev(lev).eq.10 ) then + zlev(lev).eq.100 .or. zlev(lev).eq.90 .or. & + zlev(lev).eq.80 .or. zlev(lev).eq.70 .or. & + zlev(lev).eq.60 .or. zlev(lev).eq.50 .or. & + zlev(lev).eq.40 .or. zlev(lev).eq.30 .or. & + zlev(lev).eq.20 .or. zlev(lev).eq.10 .or. & + zlev(lev).eq.9 .or. zlev(lev).eq.8 .or. & + zlev(lev).eq.7 .or. zlev(lev).eq.6 .or. & + zlev(lev).eq.5 .or. zlev(lev).eq.4 .or. & + zlev(lev).eq.3 .or. zlev(lev).eq.2 .or. & + zlev(lev).eq.1 ) then write(6,1005) int(zlev(lev)),trim(regions(iregion)),& trim(fields_3d(n)%name),nymd,nhms,hour,& rms(iregion,lev,nfield,nt,1),& diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 8481707..cf95f08 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -121,9 +121,11 @@ def remap(self): NPE = 160 PARTITION ='' - QOS = config['slurm_pbs']['qos'] + QOS ='' + + qos = config['slurm_pbs']['qos'] TIME = "1:00:00" - if QOS != "debug": TIME="12:00:00" + if qos != "debug": TIME="3:00:00" NNODE = '' job = '' @@ -131,6 +133,8 @@ def remap(self): job = "PBS" CONSTRAINT = 'cas_ait' NNODE = (NPE-1)//40 + 1 + if (qos != ''): + QOS = "#PBS -q "+qos else: job = "SLURM" partition = config['slurm_pbs']['partition'] @@ -140,8 +144,11 @@ def remap(self): CONSTRAINT = '"[cas|sky]"' if BUILT_ON_SLES15: CONSTRAINT = 'mil' + if (qos != ''): + QOS = "#SBATCH --qos="+qos account = config['slurm_pbs']['account'] + # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) suffix = '_rst.' + suffix out_rstfile = expid + os.path.basename(in_rstfile).split('_rst')[0].split('.')[-1]+suffix @@ -362,7 +369,7 @@ def has_catch_rst(text): "type": "text", "name": "slurm_pbs:qos", "message": message_qos, - "default": "debug", + "default": "", }, { diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 9b59632..46daf38 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -67,6 +67,7 @@ def parse_args(program_description): # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) + p_command.add_argument('-nonhydrostatic', action='store_true', help=" non hydrostatic upper air") p_command.add_argument('-nobkg', action='store_true', help="Do not remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") @@ -76,10 +77,13 @@ def parse_args(program_description): p_command.add_argument('-out_bc_base',default="", help="Boundary conditions base dir (w/o bc_version and resolution info) for new restarts") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") - p_command.add_argument('-qos', default="debug", help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal']) + p_command.add_argument('-qos', default="", help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal']) account = get_account() p_command.add_argument('-account', default=account, help="slurm_pbs account") - p_command.add_argument('-partition', default='', help="slurm_pbs partition") + if (BUILT_ON_SLES15): + p_command.add_argument('-partition', default='scutest',help="slurm_pbs partition") + else: + p_command.add_argument('-partition', default='', help="slurm_pbs partition") p_command.add_argument('-rs', default='3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts @@ -124,6 +128,7 @@ def get_answers_from_command_line(cml): answers["input:shared:stretch"] = cml.in_stretch answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv + answers["input:air:hydrostatic"] = not cml.nonhydrostatic if cml.rs == '1': answers["output:air:remap"] = True answers["output:surface:remap_water"] = False diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 1e7f655..4049eed 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -7,7 +7,7 @@ input: air: drymass: 1 - hydrostatic: 0 + hydrostatic: true shared: MERRA-2: false stretch: false @@ -62,5 +62,5 @@ output: lcv: false slurm_pbs: account: - qos: + qos: '' partition: '' diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index af624c0..f97712f 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -37,7 +37,10 @@ def echo_bcs(x,opt): def default_partition(x): if x['slurm_pbs:qos'] == 'debug': - x['slurm_pbs:partition'] = 'compute' + if (BUILT_ON_SLES15): + x['slurm_pbs:partition'] = 'scutest' + else: + x['slurm_pbs:partition'] = 'compute' return False return True @@ -89,6 +92,13 @@ def ask_questions(): "validate": lambda text: validate_merra2_time(text) , "when": lambda x: x['input:shared:MERRA-2'], }, + { + "type": "confirm", + "name": "input:air:hydrostatic", + "message": "Is the upper air input hydrostatic? (If you are not sure, don't change the default 'True')\n", + "default": True, + "when": lambda x: not x['input:shared:MERRA-2'], + }, { "type": "path", "name": "output:shared:out_dir", @@ -412,6 +422,7 @@ def ask_questions(): "name": "slurm_pbs:partition", "message": message_partition, "default": '', + "when": lambda x : default_partition(x), }, ] answers = questionary.prompt(questions) @@ -424,6 +435,7 @@ def ask_questions(): answers["output:surface:remap_water"] = answers["output:surface:remap"] answers["output:surface:remap_catch"] = answers["output:surface:remap"] del answers["output:surface:remap"] + if answers["input:shared:MERRA-2"] : answers["input:air:hydrostatic"] = True return answers diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 6704d8d..c699e9e 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -153,22 +153,27 @@ def remap(self): NPE = 5400; nwrit = 6 PARTITION ='' - QOS = config['slurm_pbs']['qos'] + QOS ='' + qos = config['slurm_pbs']['qos'] TIME = "1:00:00" if NPE > 532: assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS" - TIME = "12:00:00" + TIME = "3:00:00" NNODE = '' job='' if GEOS_SITE == "NAS": CONSTRAINT = 'cas_ait' NNODE = (NPE-1)//40 + 1 job='PBS' + if (qos != ''): + QOS = "#PBS -q "+qos else: job='SLURM' partition = config['slurm_pbs']['partition'] if (partition != ''): PARTITION = "#SBATCH --partition=" + partition + if (qos != ''): + QOS = "#SBATCH --qos="+qos CONSTRAINT = '"[cas|sky]"' if BUILT_ON_SLES15: @@ -179,7 +184,7 @@ def remap(self): # to interp_restarts.x. Per the code we use: # -stretched_grid target_lon target_lat stretch_fac # If we are not running stretched grid, we should pass in a blank string - stretch = config['output']['shared']['stretch'] + stretch = config['input']['shared']['stretch'] stretch_str = "" if stretch: if stretch == 'SG001': @@ -195,7 +200,24 @@ def remap(self): # note "reversed" order of args (relative to order in definition of STRETCH_GRID) - stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) + stretch_str = "-stretched_grid_in " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) + + stretch = config['output']['shared']['stretch'] + if stretch: + if stretch == 'SG001': + stretch_fac = STRETCH_GRID['SG001'][0] + target_lat = STRETCH_GRID['SG001'][1] + target_lon = STRETCH_GRID['SG001'][2] + elif stretch == 'SG002': + stretch_fac = STRETCH_GRID['SG002'][0] + target_lat = STRETCH_GRID['SG002'][1] + target_lon = STRETCH_GRID['SG002'][2] + else: + exit("This stretched grid option is not supported " + str(stretch)) + + # note "reversed" order of args (relative to order in definition of STRETCH_GRID) + + stretch_str = stretch_str+" -stretched_grid_out " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) # Now, let's create the input.nml file # We need to create a namelist for the upper air remapping @@ -218,9 +240,6 @@ def remap(self): cd {out_dir}/upper_data /bin/touch input.nml -# The MERRA fvcore_internal_restarts don't include W or DZ, but we can add them by setting -# HYDROSTATIC = 0 which means HYDROSTATIC = FALSE - if ($?I_MPI_ROOT) then # intel scaling suggestions #-------------------------- @@ -255,7 +274,14 @@ def remap(self): set ioflag = "" endif -set drymassFLG = {drymassFLG} +set hydrostaticIN = {hydrostatic} +if ( $hydrostaticIN == 'True' ) then + set hydrostaticflag = "-in_hydrostatic T" +else + set hydrostaticflag = "-in_hydrostatic F" +endif + +set drymassFLG = {drymass} if ($drymassFLG) then set dmflag = "" else @@ -263,20 +289,20 @@ def remap(self): endif {Bin}/esma_mpirun -np {NPE} $interp_restartsX -im {imout} -lm {nlevel} \\ - -do_hydro {hydrostatic} $ioflag $dmflag -nwriter {nwrit} {stretch_str} + $hydrostaticflag {stretch_str} $dmflag -nwriter {nwrit} $ioflag """ account = config['slurm_pbs']['account'] - drymassFLG = config['input']['air']['drymass'] + drymass = config['input']['air']['drymass'] hydrostatic = config['input']['air']['hydrostatic'] nlevel = config['output']['air']['nlevel'] log_name = out_dir+'/remap_upper_log' job_name = 'remap_upper' remap_upper_script = remap_template.format(Bin=bindir, account = account, \ - out_dir = out_dir, log_name = log_name, job_name= job_name, drymassFLG = drymassFLG, \ + out_dir = out_dir, log_name = log_name, job_name= job_name, drymass = drymass, \ imout = imout, nwrit = nwrit, NPE = NPE, NNODE = NNODE, \ - QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic, - stretch_str = stretch_str) + QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, \ + hydrostatic = hydrostatic, stretch_str = stretch_str) script_name = './remap_upper.j' diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 4cc6bba..5ecc1b4 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -117,7 +117,10 @@ message_ogrid_in = "Select data ocean grid/resolution of input restarts:\n" -message_qos = "SLURM or PBS quality-of-service (qos)? (If resolution is c1440 or higher, enter 'allnccs' for NCCS or 'normal' for NAS.)\n" +message_qos = """SLURM or PBS quality-of-service (qos)? + (Use default 'debug' to get resource faster; or + Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher; + or leave it blank)\n""" message_account = "Select/enter SLURM or PBS account:\n" @@ -129,20 +132,20 @@ #SBATCH --ntasks={NPE} #SBATCH --job-name={job_name} #SBATCH --output={log_name} -#SBATCH --qos={QOS} #SBATCH --time={TIME} #SBATCH --constraint={CONSTRAINT} {PARTITION} +{QOS} """, "PBS": """#!/bin/csh -f #PBS -l walltime={TIME} #PBS -l select={NNODE}:ncpus=40:mpiprocs=40:model={CONSTRAINT} #PBS -N {job_name} -#PBS -q {QOS} #PBS -W group_list={account} #PBS -o {log_name} #PBS -j oe {PARTITION} +{QOS} """ } @@ -417,7 +420,7 @@ def get_command_line_from_answers(answers): nobkg = '' if answers["output:analysis:bkg"] else " -nobkg " nolcv = '' if answers["output:analysis:lcv"] else " -nolcv " - + nonhydrostatic = '' if answers["input:air:hydrostatic"] else " -nonhydrostatic " label = ' -lbl ' if answers["output:shared:label"] else "" in_bc_base = ' -in_bc_base ' + answers.get("input:shared:bc_base") @@ -448,7 +451,9 @@ def get_command_line_from_answers(answers): noagcm_import_rst = '' if answers["output:air:agcm_import_rst"] else " -noagcm_import_rst " account = " -account " + answers["slurm_pbs:account"] - qos = " -qos " + answers["slurm_pbs:qos"] + qos = '' + if answers["slurm_pbs:qos"] != '': + qos = " -qos " + answers["slurm_pbs:qos"] partition = '' if answers["slurm_pbs:partition"] != '': partition = " -partition " + answers["slurm_pbs:partition"] @@ -476,6 +481,7 @@ def get_command_line_from_answers(answers): wemout + \ label + \ nobkg + \ + nonhydrostatic + \ noagcm_import_rst + \ nolcv + \ out_rs + \ @@ -617,5 +623,5 @@ def remove_ogrid_comment(x, opt): return False if __name__ == '__main__' : - config = yaml_to_config('c24Toc12.yaml') + config = yaml_to_config('remap_params.tpl') print_config(config)