Skip to content

Commit be34828

Browse files
nathanielerowenathanielerowe
authored
Refactor copyToSciSoft script and update version
Refactor command-line argument handling and improve script logic. Update version number and enhance debugging output.
1 parent 1e9b985 commit be34828

1 file changed

Lines changed: 161 additions & 99 deletions

File tree

‎sbndcode_wiki/attachments/copyToSciSoft‎

Lines changed: 161 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -36,26 +36,28 @@ my $iter = -1;
3636
my $machine = "scisoftgpvm01.fnal.gov";
3737
my $print_format_description=0;
3838
my $username;
39+
my $bundle_stub;
40+
my $bundle_version_stub;
3941
print_version();
4042
while ( $iter < $#ARGV ) {
4143
++$iter;
42-
if ( $ARGV[$iter] eq "-h" ) {
44+
if ( $ARGV[$iter] eq "-h" ) {
4345
print_usage();
4446
exit 0;
45-
} elsif ( $ARGV[$iter] eq "-d" ) {
47+
} elsif ( $ARGV[$iter] eq "-d" ) {
4648
$debug = 1;
4749
print "Debug switch is ON\n";
48-
} elsif ( $ARGV[$iter] eq "-n" ) {
50+
} elsif ( $ARGV[$iter] eq "-n" ) {
4951
$not_dry_run = 0;
50-
} elsif ( $ARGV[$iter] eq "--local" ) {
52+
} elsif ( $ARGV[$iter] eq "--local" ) {
5153
undef $machine;
52-
} elsif ( $ARGV[$iter] eq "--node" ) {
54+
} elsif ( $ARGV[$iter] eq "--node" ) {
5355
++$iter;
5456
$machine = $ARGV[$iter];
55-
} elsif ( $ARGV[$iter] eq "--user" ) {
57+
} elsif ( $ARGV[$iter] eq "--user" ) {
5658
++$iter;
5759
$username = $ARGV[$iter];
58-
} else {
60+
} else {
5961
if( $debug ) { print "found file $ARGV[$iter]\n"; }
6062
push (@filelist, ( $ARGV[$iter] ));
6163
}
@@ -73,10 +75,12 @@ foreach my $file (@filelist) {
7375
$copyToSciSoft::fullname = $file;
7476
$copyToSciSoft::filename = basename($copyToSciSoft::fullname);
7577
my @fileparts = split( /\./, $copyToSciSoft::filename );
76-
my $ext = (pop @fileparts) || undef;
78+
my $ext = ($#fileparts) ? (pop @fileparts) : '';
79+
undef $copyToSciSoft::bundle_stub;
80+
undef $copyToSciSoft::bundle_version_stub;
7781
if( $debug ) { print "DEBUG: checking $copyToSciSoft::filename with extension $ext\n"; }
78-
if (not $ext) {
79-
copy_script();
82+
if ( $copyToSciSoft::filename eq "index.html" ) {
83+
print "$thisfile: ignoring $copyToSciSoft::filename\n";
8084
} elsif ($ext =~ m&^t?(bz2|gz|xz|zstd|sha256)$&) {
8185
copy_archive_data();
8286
} elsif ($ext eq "txt") {
@@ -85,15 +89,17 @@ foreach my $file (@filelist) {
8589
copy_html();
8690
} elsif ($copyToSciSoft::filename =~ m&\b(buildcfg|-cfg)\b&) {
8791
copy_buildcfg();
92+
} elsif (not $ext) {
93+
copy_script();
8894
} else {
8995
print "$thisfile: ignoring $copyToSciSoft::filename\n";
9096
}
9197
}
9298

93-
if( $print_format_description ) {
99+
if( $print_format_description ) {
94100
print "\n";
95101
print "NOTICE: $thisfile has ignored one or more tarballs\n";
96-
print_format();
102+
print_format();
97103
}
98104

99105
exit 0;
@@ -120,7 +126,7 @@ sub print_usage {
120126
}
121127

122128
sub print_version {
123-
print "copyToSciSoft 1.06.00\n";
129+
print "copyToSciSoft 1.07.05\n";
124130
print "\n";
125131
}
126132

@@ -163,29 +169,74 @@ sub copy_to_scisoft {
163169
print "DEBUG: can read SciSoft directory \"$mountdir\"\n";
164170
}
165171
my $scidir = sprintf("$mountdir/%s", shift);
166-
my (@dir_exists_cmd, @mkdir_cmd, @file_exists_cmd, @file_copy_cmd, $dest);
172+
my $scidir2 = dirname($scidir);
173+
my $make_index = sprintf("$mountdir/%s", "bundles/tools/makeSciSoftIndex");
174+
my (@dir_exists_cmd, @dir_exists_cmd2, @mkdir_cmd, @file_exists_cmd, @file_copy_cmd,
175+
@index_cmd, @index_cmd2, @file_index_cmd, $dest);
167176
my $source = "$copyToSciSoft::fullname";
177+
my $bundle_dir;
178+
my $bundle_version_dir;
179+
if ( $copyToSciSoft::bundle_stub ) {
180+
$bundle_dir = $mountdir."/".$copyToSciSoft::bundle_stub;
181+
$bundle_version_dir = $mountdir."/".$copyToSciSoft::bundle_version_stub;
182+
if ($debug) { print "DEBUG: checking bundle directory $bundle_dir for $bundle_version_dir\n"; }
183+
}
168184
if ( $machine ) {
169185
$dest = "${machine}:${scidir}/";
170186
@dir_exists_cmd = ('ssh', '-K', $machine, "[[ -d \"$scidir\" ]]");
171187
@mkdir_cmd = ('ssh', '-K', $machine, "mkdir -p \"$scidir\"");
188+
@index_cmd = ('ssh', '-K', $machine, "$make_index \"$scidir2\"");
189+
@file_index_cmd = ('ssh', '-K', $machine, "$make_index \"$scidir\"");
172190
@file_exists_cmd = ('ssh', '-K', $machine, "[[ -f \"$scidir/$copyToSciSoft::filename\" ]]");
173191
@file_copy_cmd = ('scp', '-o', 'GSSAPIDelegateCredentials yes',
174192
'-p', $debug ? () : "-q", "$source", "$dest");
193+
if ( $copyToSciSoft::bundle_stub ) {
194+
@dir_exists_cmd2 = ('ssh', '-K', $machine, "[[ -d \"$bundle_version_dir\" ]]");
195+
@index_cmd2 = ('ssh', '-K', $machine, "$make_index \"$bundle_dir\"");
196+
}
175197
} else {
176198
$dest = "${scidir}/";
177199
@dir_exists_cmd = ('test', '-d', $scidir);
178200
@mkdir_cmd = ('mkdir', '-p', $scidir);
201+
@index_cmd = ("$make_index", $scidir2);
202+
@file_index_cmd = ("$make_index", $scidir);
179203
@file_exists_cmd = ('test', '-f', "$scidir/$copyToSciSoft::filename");
180204
@file_copy_cmd = ('cp', '-p', "$source", "$dest");
205+
if ( $copyToSciSoft::bundle_stub ) {
206+
@dir_exists_cmd2 = ('test', '-d', $bundle_version_dir);
207+
@index_cmd2 = ("$make_index", $bundle_dir);
208+
}
181209
}
182210
if ($not_dry_run) {
183-
debug_print_cmd(@mkdir_cmd);
184-
system(@mkdir_cmd);
185-
my $mkdir_status = last_exit_status();
186-
$debug and print "DEBUG: ", cmd_string(@mkdir_cmd), " finished with status $mkdir_status\n";
187-
if ( $mkdir_status != 0 ) {
188-
die "copy_to_scisoft: failed to create $scidir";
211+
if (system(@dir_exists_cmd) != 0) {
212+
# check to see if the version directory exists before we run mkdir
213+
my $want_bundle_index;
214+
if (($bundle_dir) && (system(@dir_exists_cmd2) != 0)) {
215+
$want_bundle_index = 1;
216+
}
217+
debug_print_cmd(@mkdir_cmd);
218+
system(@mkdir_cmd);
219+
my $mkdir_status = last_exit_status();
220+
$debug and print "DEBUG: ", cmd_string(@mkdir_cmd), " finished with status $mkdir_status\n";
221+
if ( $mkdir_status != 0 ) {
222+
die "copy_to_scisoft: failed to create $scidir";
223+
}
224+
debug_print_cmd(@index_cmd);
225+
system(@index_cmd);
226+
my $index_status = last_exit_status();
227+
$debug and print "DEBUG: ", cmd_string(@index_cmd), " finished with status $index_status\n";
228+
if ( $index_status != 0 ) {
229+
die "copy_to_scisoft: failed to create index for $scidir2";
230+
}
231+
if ($want_bundle_index) {
232+
debug_print_cmd(@index_cmd2);
233+
system(@index_cmd2);
234+
my $index_status = last_exit_status();
235+
$debug and print "DEBUG: ", cmd_string(@index_cmd2), " finished with status $index_status\n";
236+
if ( $index_status != 0 ) {
237+
die "copy_to_scisoft: failed to create index for $bundle_dir";
238+
}
239+
}
189240
}
190241
# now check to see if the file exists
191242
debug_print_cmd(@file_exists_cmd);
@@ -194,29 +245,45 @@ sub copy_to_scisoft {
194245
} else {
195246
$debug and print cmd_string(@file_copy_cmd), "\n";
196247
print "$source -> $dest ... ", $debug ? "\n" : "";
197-
if ( system(@file_copy_cmd) ) {
248+
if ( system(@file_copy_cmd) ) {
198249
my $cmd_status = last_exit_status();
199-
print "FAILED with status ", $cmd_status, "\n";
200-
exit($cmd_status);
201-
} else {
250+
print "FAILED with status ", $cmd_status, "\n";
251+
exit($cmd_status);
252+
} else {
202253
if ($debug) {
203-
printf "DEBUG: copy exited with value %d\n", last_exit_status();
254+
printf "DEBUG: copy exited with value %d\n", last_exit_status();
204255
} else {
205-
print "OK\n";
256+
print "OK\n";
206257
}
207-
}
258+
debug_print_cmd(@file_index_cmd);
259+
system(@file_index_cmd);
260+
my $index_status = last_exit_status();
261+
$debug and print "DEBUG: ", cmd_string(@file_index_cmd), " finished with status $index_status\n";
262+
if ( $index_status != 0 ) {
263+
die "copy_to_scisoft: failed to create index for $scidir";
264+
}
265+
}
208266
}
209267
} else {
210268
my $want_copy;
269+
my $want_bundle_index;
211270
if (system(@dir_exists_cmd) != 0) {
271+
if (($bundle_dir) && (system(@dir_exists_cmd2) != 0)) {
272+
$want_bundle_index = 1;
273+
}
212274
print "DRY_RUN: would execute ", cmd_string(@mkdir_cmd), "\n";
275+
print "DRY_RUN: would execute ", cmd_string(@index_cmd), "\n";
213276
$want_copy = 1;
214277
} else {
215278
debug_print_cmd(@file_exists_cmd);
216279
$want_copy = (system(@file_exists_cmd) != 0);
217280
}
281+
if ($want_bundle_index) {
282+
print "DRY_RUN: would execute ", cmd_string(@index_cmd2), "\n";
283+
}
218284
if ($want_copy) {
219-
print "DRY_RUN: would execute ", cmd_string(@file_copy_cmd), "\n"
285+
print "DRY_RUN: would execute ", cmd_string(@file_copy_cmd), "\n";
286+
print "DRY_RUN: would execute ", cmd_string(@file_index_cmd), "\n";
220287
} else {
221288
print "$copyToSciSoft::filename is already on SciSoft\n";
222289
}
@@ -248,10 +315,10 @@ sub copy_archive_data {
248315
if( $debug ) { print "DEBUG: ignoring $copyToSciSoft::filename\n"; }
249316
} elsif ( $tparts[0] eq "larsoft_suite" ) {
250317
if( $debug ) { print "DEBUG: ignoring $copyToSciSoft::filename\n"; }
251-
} elsif (( index( $tparts[0], 'genie_phyopt' ) == 0 ) && ( $tparts[0] ne "genie_phyopt")) {
318+
} elsif (( index( $tparts[0], 'genie_phyopt' ) == 0 ) && ( $tparts[0] ne "genie_phyopt")) {
252319
if( $debug ) { print "DEBUG: ignoring improperly formatted $copyToSciSoft::filename\n"; }
253320
$print_format_description=1;
254-
} elsif (( index( $tparts[0], 'genie_xsec' ) == 0 ) && ( $tparts[0] ne "genie_xsec")) {
321+
} elsif (( index( $tparts[0], 'genie_xsec' ) == 0 ) && ( $tparts[0] ne "genie_xsec")) {
255322
if( $debug ) { print "DEBUG: ignoring improperly formatted $copyToSciSoft::filename\n"; }
256323
$print_format_description=1;
257324
} else {
@@ -277,6 +344,8 @@ sub copy_html {
277344
my $version=$parts[1];
278345
$version =~ s/\.html//;
279346
my $scisoftdir="bundles/".$project."/".$version;
347+
$copyToSciSoft::bundle_stub="bundles/".$project;
348+
$copyToSciSoft::bundle_version_stub=$scisoftdir;
280349
copy_to_scisoft( $scisoftdir );
281350
}
282351

@@ -290,6 +359,8 @@ sub copy_manifest {
290359
$dotver =~ s/\./_/g;
291360
my $version = "v".$dotver;
292361
my $scisoftdir="bundles/".$product."/".$version."/manifest";
362+
$copyToSciSoft::bundle_stub="bundles/".$product;
363+
$copyToSciSoft::bundle_version_stub="bundles/".$product."/".$version;
293364
copy_to_scisoft( $scisoftdir );
294365
} else {
295366
print "$thisfile: $copyToSciSoft::filename is not a manifest\n";
@@ -305,6 +376,8 @@ sub copy_buildcfg {
305376
$dotver =~ s/\./_/g;
306377
my $version = "v".$dotver;
307378
my $scisoftdir="bundles/".$product."/".$version."/buildcfg";
379+
$copyToSciSoft::bundle_stub="bundles/".$product;
380+
$copyToSciSoft::bundle_version_stub="bundles/".$product."/".$version;
308381
copy_to_scisoft( $scisoftdir );
309382
}
310383

@@ -314,78 +387,67 @@ sub copy_script {
314387
# scripts are of the form name-version
315388
my $script=$parts[0];
316389
my $version=$parts[1];
317-
if ( $copyToSciSoft::filename eq "pullProducts" ) {
318-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
319-
my $scisoftdir="bundles/tools";
320-
copy_to_scisoft( $scisoftdir );
321-
} elsif ( $copyToSciSoft::filename eq "buildFW" ) {
322-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
323-
my $scisoftdir="bundles/tools";
324-
copy_to_scisoft( $scisoftdir );
325-
} elsif ( $copyToSciSoft::filename eq "copyToSciSoft" ) {
326-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
327-
my $scisoftdir="bundles/tools";
328-
copy_to_scisoft( $scisoftdir );
329-
} elsif ( $copyToSciSoft::filename eq "copyFromJenkins" ) {
330-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
331-
my $scisoftdir="bundles/tools";
332-
copy_to_scisoft( $scisoftdir );
333-
} elsif ( $copyToSciSoft::filename eq "getManifestList" ) {
334-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
335-
my $scisoftdir="bundles/tools";
336-
copy_to_scisoft( $scisoftdir );
337-
} elsif ( $script eq "pullProducts" ) {
338-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
339-
my $project="art";
340-
my $scisoftdir="bundles/$project/".$version;
341-
copy_to_scisoft( $scisoftdir );
342-
} elsif ( $script eq "pullArtDaq" ) {
343-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
344-
my $project="artdaq";
345-
my $scisoftdir="bundles/$project/".$version;
346-
copy_to_scisoft( $scisoftdir );
347-
} elsif ( $script eq "pullProductsLAr" ) {
348-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
349-
my $project="larsoft";
350-
my $scisoftdir="bundles/$project/".$version;
351-
copy_to_scisoft( $scisoftdir );
352-
} elsif ( $script eq "pullToyExp" ) {
353-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
354-
my $project="tutorial";
355-
my $scisoftdir="bundles/$project/".$version;
356-
copy_to_scisoft( $scisoftdir );
357-
} elsif ( $script eq "buildFW" ) {
358-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
359-
my $project="art";
360-
my $scisoftdir="bundles/$project/".$version;
361-
copy_to_scisoft( $scisoftdir );
362-
} elsif ( $script eq "buildArtDaq" ) {
363-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
364-
my $project="artdaq";
365-
my $scisoftdir="bundles/$project/".$version;
366-
copy_to_scisoft( $scisoftdir );
367-
} elsif ( $script eq "buildLAr" ) {
368-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
369-
my $project="larsoft";
370-
my $scisoftdir="bundles/$project/".$version;
371-
copy_to_scisoft( $scisoftdir );
372-
} elsif ( $script eq "buildToyExp" ) {
373-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
374-
my $project="tutorial";
375-
my $scisoftdir="bundles/$project/".$version;
376-
copy_to_scisoft( $scisoftdir );
377-
} elsif ( $script eq "checkPrerequisites" ) {
378-
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
379-
my $project="art";
380-
my $scisoftdir="bundles/$project/".$version;
381-
copy_to_scisoft( $scisoftdir );
382-
} elsif ( index( $script, 'pull' ) == 0 ) {
390+
if ( $copyToSciSoft::filename eq "pullProducts" ) {
391+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
392+
my $scisoftdir="bundles/tools";
393+
copy_to_scisoft( $scisoftdir );
394+
} elsif ( $copyToSciSoft::filename eq "buildFW" ) {
395+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
396+
my $scisoftdir="bundles/tools";
397+
copy_to_scisoft( $scisoftdir );
398+
} elsif ( $copyToSciSoft::filename eq "copyToSciSoft" ) {
399+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
400+
my $scisoftdir="bundles/tools";
401+
copy_to_scisoft( $scisoftdir );
402+
} elsif ( $copyToSciSoft::filename eq "copyFromJenkins" ) {
403+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
404+
my $scisoftdir="bundles/tools";
405+
copy_to_scisoft( $scisoftdir );
406+
} elsif ( $copyToSciSoft::filename eq "makeSciSoftIndex" ) {
407+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
408+
my $scisoftdir="bundles/tools";
409+
copy_to_scisoft( $scisoftdir );
410+
} elsif ( $copyToSciSoft::filename eq "getManifestList" ) {
411+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
412+
my $scisoftdir="bundles/tools";
413+
copy_to_scisoft( $scisoftdir );
414+
} elsif ( $script eq "pullProducts" ) {
415+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::fullname script\n"; }
416+
my $project="art";
417+
my $scisoftdir="bundles/$project/".$version;
418+
copy_to_scisoft( $scisoftdir );
419+
} elsif ( $script eq "pullArtDaq" ) {
420+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
421+
my $project="artdaq";
422+
my $scisoftdir="bundles/$project/".$version;
423+
copy_to_scisoft( $scisoftdir );
424+
} elsif ( $script eq "pullProductsLAr" ) {
425+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
426+
my $project="larsoft";
427+
my $scisoftdir="bundles/$project/".$version;
428+
copy_to_scisoft( $scisoftdir );
429+
} elsif ( $script eq "pullToyExp" ) {
430+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
431+
my $project="tutorial";
432+
my $scisoftdir="bundles/$project/".$version;
433+
copy_to_scisoft( $scisoftdir );
434+
} elsif ( $script eq "buildFW" ) {
435+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
436+
my $project="art";
437+
my $scisoftdir="bundles/$project/".$version;
438+
copy_to_scisoft( $scisoftdir );
439+
} elsif ( $script eq "buildArtDaq" ) {
440+
if( $debug ) { print "DEBUG: copy $copyToSciSoft::filename script\n"; }
441+
my $project="artdaq";
442+
my $scisoftdir="bundles/$project/".$version;
443+
copy_to_scisoft( $scisoftdir );
444+
} elsif ( index( $script, 'pull' ) == 0 ) {
383445
print "$thisfile: ignoring $copyToSciSoft::filename script\n";
384-
} elsif ( index( $script, 'build' ) == 0 ) {
446+
} elsif ( index( $script, 'build' ) == 0 ) {
385447
print "$thisfile: ignoring $copyToSciSoft::filename script\n";
386-
} elsif ( index( $script, 'check' ) == 0 ) {
448+
} elsif ( index( $script, 'check' ) == 0 ) {
387449
print "$thisfile: ignoring $copyToSciSoft::filename script\n";
388-
} else {
450+
} else {
389451
print "$thisfile: ignoring $copyToSciSoft::filename\n";
390452
#my $scisoftdir="manifest/".$script."/".$version;
391453
#copy_to_scisoft( $scisoftdir );

0 commit comments

Comments
 (0)