prrte-4.1.1~rc2/0000775000175000017500000000000015203542143013641 5ustar alastairalastairprrte-4.1.1~rc2/VERSION0000664000175000017500000000704715203542132014717 0ustar alastairalastair# Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. # Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2011 NVIDIA Corporation. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. # Copyright (c) 2016 IBM Corporation. All rights reserved. # Copyright (c) 2017 Los Alamos National Security, LLC. All rights # reserved. # This is the VERSION file for PRRTE, describing the precise # version of PRRTE in this distribution. The various components of # the version number below are combined to form a single version # number string. # major, minor, and release are generally combined in the form # ... major=4 minor=1 release=1 # PRRTE required dependency versions. # List in x.y.z format. pmix_min_version=6.1.0 pmix_max_version=1000.0.0 hwloc_min_version=2.1.0 event_min_version=2.0.21 automake_min_version=1.13.4 autoconf_min_version=2.69.0 libtool_min_version=2.4.2 flex_min_version=2.5.4 python_min_version=3.6 # greek is generally used for alpha or beta release tags. If it is # non-empty, it will be appended to the version number. It does not # have to be numeric. Common examples include: a1 (alpha release 1), # b1 (beta release 1), rc2 (release candidate 2). The only # requirement is that it must be entirely printable ASCII characters # and have no white space. greek=rc2 # If repo_rev is empty, then the repository version number will be # obtained during "make dist" via the "git describe --tags --always" # command, or with the date (if "git describe" fails) in the form of # "date". repo_rev= # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in # this file. For example, if tarball_version is empty, the tarball # filename will be of the form # openmpi-...tar.*. However, if # tarball_version is not empty, the tarball filename will be of the # form openmpi-.tar.*. tarball_version=gitclone # The date when this release was created date="N/A" # The shared library version of each of PRRTE's public libraries. # These versions are maintained in accordance with the "Library # Interface Versions" chapter from the GNU Libtool documentation: # # - If the library source code has changed at all since the last # update, then increment revision (`c:r:a' becomes `c:r+1:a'). # # - If any interfaces have been added, removed, or changed since # the last update, increment current, and set revision to 0. # # - If any interfaces have been added since the last public release, # then increment age. # # - If any interfaces have been removed since the last public release, # then set age to 0. # # All changes in these version numbers are dictated by the PMIx # release managers (not individual developers). Notes: # 1. Since these version numbers are associated with *releases*, the # version numbers maintained on the PMIx Github trunk (and developer # branches) is always 0:0:0 for all libraries. # 2. The version number of libpmix refers to the public pmix interfaces. # It does not refer to any internal interfaces. # Version numbers are described in the Libtool current:revision:age # format. libprrte_so_version=5:1:1 # "Common" components install standalone libraries that are run-time # linked by one or more components. So they need to be versioned as # well. Yuck; this somewhat breaks the # components-don't-affect-the-build-system abstraction. prrte-4.1.1~rc2/autogen.pl0000775000175000017500000011137115203542132015645 0ustar alastairalastair#!/usr/bin/env perl # # Copyright (c) 2009-2022 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. # Copyright (c) 2013-2020 Intel, Inc. All rights reserved. # Copyright (c) 2015 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2015 IBM Corporation. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # Copyright (c) 2023 Jeffrey M. Squyres. All rights reserved. # # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # use strict; use Cwd; use File::Basename; use File::Find; use Data::Dumper; use Getopt::Long; # # Global variables # # Sentinel file to remove if we fail my $sentinel; # The m4 file we'll write at the end my $m4_output_file = "config/autogen_found_items.m4"; my $m4; # Sanity check file my $topdir_file = "docs/index.rst"; my $dnl_line = "dnl ---------------------------------------------------------------------------"; # The text file we'll write at the end that will contain # all the mca component directory paths my $mca_library_paths_file = "config/mca_library_paths.txt"; # Data structures to fill up with all the stuff we find my $mca_found; my @subdirs; # Command line parameters my $quiet_arg = 0; my $debug_arg = 0; my $help_arg = 0; my $include_arg = 0; my $exclude_arg = 0; my $force_arg = 0; # Include/exclude lists my $include_list; my $exclude_list; # Minimum versions my $prte_automake_version = "1.13.4"; my $prte_autoconf_version = "2.69"; my $prte_libtool_version = "2.4.2"; # Search paths my $prte_autoconf_search = "autoconf"; my $prte_automake_search = "automake"; my $prte_libtoolize_search = "libtoolize;glibtoolize"; # One-time setup my $username; my $hostname; my $full_hostname; # Patch program my $patch_prog = "patch"; $username = getpwuid($>); $full_hostname = `hostname`; chomp($full_hostname); $hostname = $full_hostname; $hostname =~ s/^([\w\-]+)\..+/\1/; ############################################################################## sub my_die { unlink($sentinel) if ($sentinel); die @_; } sub my_exit { my ($ret) = @_; unlink($sentinel) if ($sentinel && $ret != 0); exit($ret); } ############################################################################## sub verbose { print @_ if (!$quiet_arg); } sub debug { print @_ if ($debug_arg); } sub debug_dump { my $d = new Data::Dumper([@_]); $d->Purity(1)->Indent(1); debug $d->Dump; } ############################################################################## sub mca_process_component { my ($framework, $component) = @_; my $cdir = "src/mca/$framework/$component"; return if (! -d $cdir); # Process this directory my $found_component; $found_component = { name => $component, framework_name => $framework, abs_dir => $cdir, }; # Does this directory have a configure.m4 file? if (-f "$cdir/configure.m4") { $found_component->{"configure.m4"} = 1; verbose " Found configure.m4 file\n"; } # Does this directory have a # help-{framework}-{component}.rst file? if (-f "$cdir/help-$framework-$component.rst") { $found_component->{"rst"} = 1; verbose " Found help-$framework-$component.rst file\n"; } # Push the results onto the $mca_found hash array push(@{$mca_found->{$framework}->{"components"}}, $found_component); # save the directory for later to create the paths # to all the component libraries push(@subdirs, $cdir); } ############################################################################## sub ignored { my ($dir) = @_; # If this directory does not have .prte_ignore, or if it has a # .prte_unignore that has my username in it, then add it to the # list of components. my $ignored = 0; if (-f "$dir/.prte_ignore") { $ignored = 1; } if (-f "$dir/.prte_unignore") { open(UNIGNORE, "$dir/.prte_unignore") || my_die "Can't open $dir/.prte_unignore file"; my $unignore; $unignore .= $_ while (); close(UNIGNORE); $ignored = 0 if ($unignore =~ /^$username$/m || $unignore =~ /^$username\@$hostname$/m || $unignore =~ /^$username\@$full_hostname$/m); } return $ignored; } ############################################################################## sub mca_process_framework { my ($framework) = @_; # Does this framework have a configure.m4 file? my $dir = "src/mca/$framework"; if (-f "$dir/configure.m4") { $mca_found->{$framework}->{"configure.m4"} = 1; verbose " Found framework configure.m4 file\n"; } # Did we exclude all components for this framework? if (exists($exclude_list->{$framework}) && $exclude_list->{$framework}[0] eq "AGEN_EXCLUDE_ALL") { verbose " => Excluded\n"; } else { # Look for component directories in this framework if (-d $dir) { $mca_found->{$framework}->{found} = 1; opendir(DIR, $dir) || my_die "Can't open $dir directory"; foreach my $d (sort(readdir(DIR))) { # Skip any non-directory, "base", or any dir that # begins with "." next if (! -d "$dir/$d" || $d eq "base" || substr($d, 0, 1) eq "."); # Skip any component that doesn't have a configure.m4 # or Makefile.am as we couldn't build it anyway if (! -f "$dir/$d/configure.m4" && ! -f "$dir/$d/Makefile.am" && ! -f "$dir/$d/configure.ac" && ! -f "$dir/$d/configure.in") { verbose " => No sentinel file found in $dir/$d -> Excluded\n"; next; } verbose "--- Found PRRTE / $framework / $d component: src/mca/$framework/$d\n"; # Skip if specifically excluded if (exists($exclude_list->{$framework})) { my $tst = 0; foreach my $ck (@{$exclude_list->{$framework}}) { if ($ck eq $d) { verbose " => Excluded\n"; $tst = 1; last; } } if ($tst) { next; } } # Skip if the framework is on the include list, but # doesn't contain this component if (exists($include_list->{$framework})) { my $tst = 0; foreach my $ck (@{$include_list->{$framework}}) { if ($ck ne $d) { verbose " => Not included\n"; $tst = 1; last; } } if ($tst) { next; } } # Check ignore status if (ignored("$dir/$d")) { verbose " => Ignored (found .prte_ignore file)\n"; } else { mca_process_component($framework, $d); } } } closedir(DIR); } } ############################################################################## sub mca_generate_framework_header(\$\@) { my (@frameworks) = @_; my $framework_array_output=""; my $framework_decl_output=""; my $framework_name_output=""; foreach my $framework (@frameworks) { # There is no common framework object if ($framework ne "common" and $framework ne "src") { my $framework_name = "prte_${framework}_base_framework"; $framework_array_output .= " &$framework_name,\n"; $framework_decl_output .= "extern pmix_mca_base_framework_t $framework_name;\n"; $framework_name_output .= " \"${framework}\",\n"; } } $framework_name_output .= " \"hwloc\",\n"; $framework_name_output .= " \"if\",\n"; $framework_name_output .= " \"reachable\",\n"; my $ifdef_string = uc "prte_FRAMEWORKS_H"; open(FRAMEWORKS_OUT, ">src/include/prte_frameworks.h"); printf FRAMEWORKS_OUT "%s", "/* * This file is autogenerated by autogen.pl. Do not edit this file by hand. */ #ifndef $ifdef_string #define $ifdef_string #include \"src/mca/base/pmix_mca_base_framework.h\" $framework_decl_output PRTE_EXPORT extern pmix_mca_base_framework_t *prte_frameworks[]; PRTE_EXPORT extern char *prte_framework_names[]; #endif /* $ifdef_string */\n\n"; close(FRAMEWORKS_OUT); open(FRAMEWORKS_OUT, ">src/include/prte_frameworks.c"); printf FRAMEWORKS_OUT "%s", "/* * This file is autogenerated by autogen.pl. Do not edit this file by hand. */ #include \"src/include/prte_config.h\" #include \"src/include/prte_frameworks.h\" pmix_mca_base_framework_t *prte_frameworks[] = { $framework_array_output NULL }; char *prte_framework_names[] = { $framework_name_output NULL }; "; close(FRAMEWORKS_OUT); } ############################################################################## sub mca_process_project { # Look for framework directories my $dir = "src/mca"; opendir(DIR, $dir) || my_die "Can't open $dir directory"; my @my_dirs = readdir(DIR); @my_dirs = sort(@my_dirs); foreach my $d (@my_dirs) { # Skip any non-directory, "base", or any dir that begins with "." next if (! -d "$dir/$d" || $d eq "base" || substr($d, 0, 1) eq "."); # If this directory has a $dir.h file and a base/ # subdirectory, or its name is "common", then it's a # framework. if ("common" eq $d || (-f "$dir/$d/$d.h" && -d "$dir/$d/base")) { verbose "\n=== Found prte framework: src/mca/$d\n"; mca_process_framework($d); } } closedir(DIR); } ############################################################################## sub mca_run_global { # Go find a list of frameworks, and for each of # those, go find a list of components. mca_process_project(); # Debugging output debug_dump($mca_found); $m4 .= "\n$dnl_line $dnl_line $dnl_line dnl MCA information\n"; # Array for all the m4_includes that we'll need to pick up the # configure.m4's. my @includes; # Write the list of frameworks my $frameworks_comma; # Print out project-level info my @mykeys = keys(%{$mca_found}); @mykeys = sort(@mykeys); # Ensure that the "common" framework is listed first # (if it exists) my @tmp; push(@tmp, "common") if (grep(/common/, @mykeys)); foreach my $f (@mykeys) { push(@tmp, $f) if ($f ne "common"); } @mykeys = @tmp; foreach my $f (@mykeys) { $frameworks_comma .= ", $f"; # Does this framework have a configure.m4 file? push(@includes, "src/mca/$f/configure.m4") if (exists($mca_found->{$f}->{"configure.m4"})); # This framework does have a Makefile.am (or at least, # it should!) my_die "Missing src/mca/$f/Makefile.am" if (! -f "src/mca/$f/Makefile.am"); } $frameworks_comma =~ s/^, //; &mca_generate_framework_header("src", @mykeys); $m4 .= "$dnl_line dnl Frameworks in the prte project and their corresponding directories m4_define([mca_prte_framework_list], [$frameworks_comma]) "; # Print out framework-level info foreach my $f (@mykeys) { my $components; my $m4_config_component_list; my $no_config_component_list; # Troll through each of the found components foreach my $comp (@{$mca_found->{$f}->{components}}) { my $c = $comp->{name}; $components .= "$c "; # Does this component have a configure.m4 file? if (exists($comp->{"configure.m4"})) { push(@includes, "src/mca/$f/$c/configure.m4"); $m4_config_component_list .= ", $c"; } else { $no_config_component_list .= ", $c"; } } $m4_config_component_list =~ s/^, //; $no_config_component_list =~ s/^, //; $m4 .= "dnl Components in the prte / $f framework m4_define([mca_prte_${f}_m4_config_component_list], [$m4_config_component_list]) m4_define([mca_prte_${f}_no_config_component_list], [$no_config_component_list]) "; } # List out all the m4_include $m4 .= "$dnl_line dnl List of configure.m4 files to include\n"; foreach my $i (@includes) { $m4 .= "m4_include([$i])\n"; } } ############################################################################## # Find and remove stale files sub find_and_delete { foreach my $file (@_) { my $removed = 0; if (-f $file) { unlink($file); $removed = 1; } if (-f "config/$file") { unlink("config/$file"); $removed = 1; } debug " Removed stale copy of $file\n" if ($removed); } } ############################################################################## # Find a specific executable and ensure that it is a recent enough # version. sub find_and_check { my ($app, $app_name, $req_version) = @_; my @search_path = split(/;/, $app_name); my @min_version = split(/\./, $req_version); my @versions_found = (); foreach (@search_path) { verbose " Searching for $_\n"; my $version = `$_ --version`; if (!defined($version)) { verbose " $_ not found\n"; next; } # Matches a version string with 1 or more parts possibly prefixed with a letter (ex: # v2.2) or followed by a letter (ex: 2.2.6b). This regex assumes there is a space # before the version string and that the version is ok if there is no version. if (!($version =~ m/\s[vV]?(\d[\d\.]*\w?)/m)) { verbose " WARNING: $_ does not appear to support --version. Assuming it is ok\n"; return; } $version = $1; verbose " Found $_ version $version; checking version...\n"; push(@versions_found, $version); my @parts = split(/\./, $version); my $i = 0; # Check every component of the version number while ($i <= $#min_version) { verbose " Found version component $parts[$i] -- need $min_version[$i]\n"; # Check to see if there are any characters (!) in the # version number (e.g., Libtool's "2.2.6b" -- #%@#$%!!!). # Do separate comparisons between the number and any # trailing digits. You can't just "lt" compare the whole # string because "10 lt 2b" will return true. #@$@#$#@$ # Libtool!! $parts[$i] =~ m/(\d+)([a-z]*)/i; my $pn = $1; my $pa = $2; $min_version[$i] =~ m/(\d+)([a-z]*)/i; my $mn = $1; my $ma = $2; # If the version is higher, we're done. if ($pn > $mn) { verbose " ==> ACCEPTED\n"; return; } # If the version is lower, we're done. elsif ($pn < $mn || ($pn == $mn && $pa lt $ma)) { verbose " ==> Too low! Skipping this version\n"; last; } # If the version was equal, keep checking. ++$i; } # If we found a good version, return. if ($i > $#min_version) { verbose " ==> ACCEPTED\n"; return; } } # if no acceptable version found, reject it print " ================================================================= I could not find a recent enough copy of $app. I need at least $req_version, but only found the following versions:\n\n"; my $i = 0; foreach (@search_path) { print " $_: $versions_found[$i]\n"; $i++; } print "\nI am gonna abort. :-( Please make sure you are using at least the following versions of the tools: GNU Autoconf: $prte_autoconf_version GNU Automake: $prte_automake_version GNU Libtool: $prte_libtool_version =================================================================\n"; my_exit(1); } ############################################################################## sub safe_system { print "Running: " . join(/ /, @_) . "\n"; my $ret = system(@_); $ret >>= 8; if (0 != $ret) { print "Command failed: @_\n"; my_exit($ret); } $ret; } ############################################################################## sub patch_autotools_output { my ($topdir) = @_; # Set indentation string for verbose output depending on current directory. my $indent_str = " "; if ($topdir eq ".") { $indent_str = "=== "; } # Patch ltmain.sh error for PGI version numbers. Redirect stderr to # /dev/null because this patch is only necessary for some versions of # Libtool (e.g., 2.2.6b); it'll [rightfully] fail if you have a new # enough Libtool that doesn't need this patch. But don't alarm the # user and make them think that autogen failed if this patch fails -- # make the errors be silent. # Also patch ltmain.sh for NAG compiler if (-f "config/ltmain.sh") { verbose "$indent_str"."Patching PGI compiler version numbers in ltmain.sh\n"; system("$patch_prog -N -p0 < $topdir/config/ltmain_pgi_tp.diff >/dev/null 2>&1"); unlink("config/ltmain.sh.rej"); verbose "$indent_str"."Patching \"-pthread\" option for NAG compiler in ltmain.sh\n"; system("$patch_prog -N -p0 < $topdir/config/ltmain_nag_pthread.diff >/dev/null 2>&1"); unlink("config/ltmain.sh.rej"); } # If there's no configure script, there's nothing else to do. return if (! -f "configure"); my @verbose_out; # Total ugh. We have to patch the configure script itself. See below # for explanations why. open(IN, "configure") || my_die "Can't open configure"; my $c; $c .= $_ while(); close(IN); my $c_orig = $c; # The PGI 10 version number broke <=LT # 2.2.6b's version number checking regexps. We can't fix the # Libtool install; all we can do is patch the resulting configure # script. :-( The following comes from the upstream patch: # http://lists.gnu.org/archive/html/libtool-patches/2009-11/msg00016.html push(@verbose_out, $indent_str . "Patching configure for Libtool PGI version number regexps\n"); $c =~ s/\*pgCC\\ \[1-5\]\* \| \*pgcpp\\ \[1-5\]\*/*pgCC\\ [1-5]\.* | *pgcpp\\ [1-5]\.*/g; # See http://git.savannah.gnu.org/cgit/libtool.git/commit/?id=v2.2.6-201-g519bf91 for details # Note that this issue was fixed in LT 2.2.8, however most distros are still using 2.2.6b push(@verbose_out, $indent_str . "Patching configure for IBM xlf libtool bug\n"); $c =~ s/(\$LD -shared \$libobjs \$deplibs \$)compiler_flags( -soname \$soname)/$1linker_flags$2/g; #Check if we are using a recent enough libtool that supports PowerPC little endian if(index($c, 'powerpc64le-*linux*)') == -1) { push(@verbose_out, $indent_str . "Patching configure for PowerPC little endian support\n"); my $replace_string = "x86_64-*kfreebsd*-gnu|x86_64-*linux*|powerpc*-*linux*|"; $c =~ s/x86_64-\*kfreebsd\*-gnu\|x86_64-\*linux\*\|ppc\*-\*linux\*\|powerpc\*-\*linux\*\|/$replace_string/g; $replace_string = "powerpc64le-*linux*)\n\t LD=\"\${LD-ld} -m elf32lppclinux\"\n\t ;;\n\t powerpc64-*linux*)"; $c =~ s/ppc64-\*linux\*\|powerpc64-\*linux\*\)/$replace_string/g; $replace_string = "powerpcle-*linux*)\n\t LD=\"\${LD-ld} -m elf64lppc\"\n\t ;;\n\t powerpc-*linux*)"; $c =~ s/ppc\*-\*linux\*\|powerpc\*-\*linux\*\)/$replace_string/g; } # Fix consequence of broken libtool.m4 # see http://lists.gnu.org/archive/html/bug-libtool/2015-07/msg00002.html and # https://github.com/open-mpi/ompi/issues/751 push(@verbose_out, $indent_str . "Patching configure for -L/-R libtool.m4 bug\n"); # patch for libtool < 2.4.3 $c =~ s/# Some compilers place space between "-\{L,R\}" and the path.\n # Remove the space.\n if test \$p = \"-L\" \|\|/# Some compilers place space between "-\{L,-l,R\}" and the path.\n # Remove the spaces.\n if test \$p = \"-L\" \|\|\n test \$p = \"-l\" \|\|/g; # patch for libtool >= 2.4.3 $c =~ s/# Some compilers place space between "-\{L,R\}" and the path.\n # Remove the space.\n if test x-L = \"\$p\" \|\|\n test x-R = \"\$p\"\; then/# Some compilers place space between "-\{L,-l,R\}" and the path.\n # Remove the spaces.\n if test x-L = \"x\$p\" \|\|\n test x-l = \"x\$p\" \|\|\n test x-R = \"x\$p\"\; then/g; # Fix OS X Big Sur (11.0.x) support # From https://lists.gnu.org/archive/html/libtool-patches/2020-06/msg00001.html push(@verbose_out, $indent_str . "Patching configure for MacOS Big Sur libtool.m4 bug\n"); # Some versions of Libtool use ${wl} consistently, but others did # not (e.g., they used $wl). Make the regexp be able to handle # both. Additionally, the case string searching for 10.[012]* # changed over time. So make sure it can handle both of the case # strings that we're aware of. my $WL = '(\$\{wl\}|\$wl)'; my $SOMETIMES = '(\[,.\])*'; my $search_string = 'darwin\*\) # darwin 5.x on # if running on 10.5 or later, the deployment target defaults # to the OS version, if on x86, and 10.4, the deployment # target defaults to 10.4. Don\'t you love it\? case \$\{MACOSX_DEPLOYMENT_TARGET-10.0\},\$host in 10.0,\*86\*-darwin8\*\|10.0,\*-darwin\[91\]\*\) _lt_dar_allow_undefined=\'' . $WL . '-undefined ' . $WL . 'dynamic_lookup\' ;; 10.\[012\]' . $SOMETIMES . '\*\) _lt_dar_allow_undefined=\'' . $WL . '-flat_namespace ' . $WL . '-undefined ' . $WL . 'suppress\' ;; 10.\*\)'; my $replace_string = 'darwin*) # PRRTE patched for Darwin / MacOS Big Sur. See # http://lists.gnu.org/archive/html/bug-libtool/2015-07/msg00001.html case ${MACOSX_DEPLOYMENT_TARGET},$host in 10.[012],*|,*powerpc*) _lt_dar_allow_undefined=\'${wl}-flat_namespace ${wl}-undefined ${wl}suppress\' ;; *)'; $c =~ s/$search_string/$replace_string/g; # Only write out verbose statements and a new configure if the # configure content actually changed return if ($c eq $c_orig); foreach my $str (@verbose_out) { verbose($str); } open(OUT, ">configure.patched") || my_die "Can't open configure.patched"; print OUT $c; close(OUT); # Use cp so that we preserve permissions on configure safe_system("cp configure.patched configure"); unlink("configure.patched"); } sub export_version { my ($name,$version) = @_; $version =~ s/[^a-zA-Z0-9,.]//g; my @version_splits = split(/\./,$version); my $hex = sprintf("0x%04x%02x%02x", $version_splits[0], $version_splits[1], $version_splits[2]); $m4 .= "m4_define([PRTE_${name}_MIN_VERSION], [$version])\n"; $m4 .= "m4_define([PRTE_${name}_NUMERIC_MIN_VERSION], [$hex])\n"; } sub export_max_version { my ($name,$version) = @_; $version =~ s/[^a-zA-Z0-9,.]//g; my @version_splits = split(/\./,$version); my $hex = sprintf("0x%04x%02x%02x", $version_splits[0], $version_splits[1], $version_splits[2]); $m4 .= "m4_define([PRTE_${name}_MAX_VERSION], [$version])\n"; $m4 .= "m4_define([PRTE_${name}_NUMERIC_MAX_VERSION], [$hex])\n"; } sub get_and_define_min_versions() { open(IN, "VERSION") || my_die "Can't open VERSION"; while () { my $line = $_; my @fields = split(/=/,$line); if ($fields[0] eq "automake_min_version") { if ($fields[1] ne "\n") { $prte_automake_version = $fields[1]; } } elsif($fields[0] eq "autoconf_min_version") { if ($fields[1] ne "\n") { $prte_autoconf_version = $fields[1]; } } elsif($fields[0] eq "libtool_min_version") { if ($fields[1] ne "\n") { $prte_libtool_version = $fields[1]; } } elsif($fields[0] eq "pmix_min_version") { if ($fields[1] ne "\n") { export_version("PMIX", $fields[1]); } } elsif($fields[0] eq "pmix_max_version") { if ($fields[1] ne "\n") { export_max_version("PMIX", $fields[1]); } } elsif($fields[0] eq "hwloc_min_version") { if ($fields[1] ne "\n") { export_version("HWLOC", $fields[1]); } } elsif($fields[0] eq "event_min_version") { if ($fields[1] ne "\n") { export_version("EVENT", $fields[1]); } } elsif($fields[0] eq "flex_min_version") { if ($fields[1] ne "\n") { export_version("FLEX", $fields[1]); } } elsif($fields[0] eq "python_min_version") { if ($fields[1] ne "\n") { export_version("PYTHON", $fields[1]); } } } close(IN); } ############################################################################## sub in_tarball { my $tarball = 0; open(IN, "VERSION") || my_die "Can't open VERSION"; # If repo_rev is not an empty string, we are in a tarball while () { my $line = $_; my @fields = split(/=/,$line); if ($fields[0] eq "repo_rev") { if ($fields[1] ne "\n") { $tarball = 1; last; } } } close(IN); return $tarball; } ############################################################################## sub replace_config_sub_guess { # This could be simpler if we could use some Perl modules for this # functionality (e.g., DateTime). But I don't want to introduce # any CPAN dependencies here, so just do sometime simple, even if # it's a bit laborious. Use a few private helper functions for # this kind of functionality. sub _get_timestamp { my $filename = shift; my $ret; if (-x $filename) { my $out = `$filename --version`; $out =~ m/GNU config\.[a-z]+ \((.+)\)/; $ret = $1; } return $ret; } sub _split_timestamp { my $ts = shift; $ts =~ m/(\d+)-(\d+)-(\d+)/; return $1, $2, $3; } # Returns true if timestamp $a > timestamp $b. sub _timestamp_gt { my ($a, $b) = @_; my ($year_a, $month_a, $day_a) = _split_timestamp($a); my ($year_b, $month_b, $day_b) = _split_timestamp($b); # Don't try to be clever -- just do a simple set of explicit # comparisons. if ($year_a > $year_b) { return 1; } elsif ($year_a < $year_b) { return 0; } else { if ($month_a > $month_b) { return 1; } elsif ($month_a < $month_b) { return 0; } else { if ($day_a > $day_b) { return 1; } else { return 0; } } } } my ($topdir) = @_; # Find the stashed known-good files, and get their version # timestamps. my $cached_dir = "$topdir/config/from-savannah"; my @files = qw/config.guess config.sub/; my %known_good_timestamps; foreach my $file (@files) { my $filename = "$cached_dir/upstream-$file"; my_die("Cannot find $filename") if (! -f $filename); my $ts = _get_timestamp($filename); $known_good_timestamps{$file} = $ts; } # Find all config.guess/config.sub files in the tree. If their # versions are older than the stashed known-good files, update # them from the stash. my @files; File::Find::find(sub { push(@files, $File::Find::name) if ($_ eq "config.guess" || $_ eq "config.sub") }, $topdir); foreach my $file (@files) { my $base = basename($file); my $ts = _get_timestamp($file); if (_timestamp_gt($known_good_timestamps{$base}, $ts)) { print("=== Replacing $file with newer version\n"); safe_system("cp -f $cached_dir/upstream-$base $file"); } } } ############################################################################## ############################################################################## ## main - do the real work... ############################################################################## ############################################################################## # Command line parameters my $ok = Getopt::Long::GetOptions("quiet|q" => \$quiet_arg, "debug|d" => \$debug_arg, "help|h" => \$help_arg, "include=s" => \$include_arg, "exclude=s" => \$exclude_arg, "force|f" => \$force_arg, ); if (!$ok || $help_arg) { print "Invalid command line argument.\n\n" if (!$ok); print "Options: --quiet | -q Do not display normal verbose output --debug | -d Output lots of debug information --help | -h This help list --include | -i Comma-separated list of framework-component pairs to be exclusively built - i.e., all other components will be ignored and only those specified will be marked to build --exclude | -e Comma-separated list of framework or framework-component to be excluded from the build --force | -f Run even if invoked from the source tree of an expanded distribution tarball\n"; my_exit($ok ? 0 : 1); } #--------------------------------------------------------------------------- # Check for project existence my $project_name_long = "prte"; my $project_name_short = "prte"; #--------------------------------------------------------------------------- $full_hostname = `hostname`; chomp($full_hostname); $m4 = "dnl dnl \$HEADER\$ dnl $dnl_line dnl This file is automatically created by autogen.pl; it should not dnl be edited by hand!! dnl dnl Generated by $username at " . localtime($ENV{SOURCE_DATE_EPOCH} || time) . " dnl on $full_hostname. $dnl_line\n\n"; #--------------------------------------------------------------------------- # Verify that we're in the PRTE root directory by checking for a token file. my_die "Not at the root directory of a PRRTE source tree" if (! -f "config/prte_mca.m4"); $force_arg = 1; my_die "autogen.pl has been invoked in the source tree of a PRRTE distribution tarball; aborting... You likely do not need to invoke \"autogen.pl\" -- you can probably run \"configure\" directly. If you really know what you are doing, and really need to run autogen.pl, use the \"--force\" flag." if (!$force_arg && in_tarball()); # Now that we've verified that we're in the top-level PRRTE directory, # set the sentinel file to remove if we abort. $sentinel = Cwd::cwd() . "/configure"; #--------------------------------------------------------------------------- my $step = 1; verbose "PRRTE autogen (buckle up!) $step. Checking tool versions\n\n"; get_and_define_min_versions(); # Check the autotools revision levels &find_and_check("autoconf", $prte_autoconf_search, $prte_autoconf_version); &find_and_check("libtool", $prte_libtoolize_search, $prte_libtool_version); &find_and_check("automake", $prte_automake_search, $prte_automake_version); #--------------------------------------------------------------------------- ++$step; verbose "\n$step. Checking for git submodules\n\n"; # Make sure we got a submodule-full clone. If not, abort and let a # human figure it out. if (-f ".gitmodules") { open(IN, "git submodule status|") || die "Can't run \"git submodule status\""; while () { $_ =~ m/^(.)[0-9a-f]{40}\s+(\S+)/; my $status = $1; my $path = $2; print("=== Submodule: $path\n"); # Make sure the submodule is there if ($status eq "-") { print(" ==> ERROR: Missing The submodule \"$path\" is missing. Perhaps you forgot to \"git clone --recursive ...\", or you need to \"git submodule update --init --recursive\"...?\n\n"); exit(1); } # See if the commit in the submodule is not the same as the # commit that the git submodule thinks it should be. elsif ($status eq "+") { print(" ==> WARNING: Submodule hash is different than upstream. If this is not intentional, you may want to run: \"git submodule update --init --recursive\"\n"); } else { print(" Local hash is what is expected by the submodule (good!)\n"); } } } #--------------------------------------------------------------------------- # No platform file -- write an empty list $m4 .= "m4_define([autogen_platform_file], [])\n\n"; if ($exclude_arg) { debug "Using exclude list: $exclude_arg"; my @list = split(/,/, $exclude_arg); foreach (@list) { my @pairs = split(/-/, $_); if (exists($pairs[1])) { # Remove any trailing newlines chomp($pairs[1]); debug " Adding ".$pairs[0]."->".$pairs[1]." to exclude list\n"; push(@{$exclude_list->{$pairs[0]}}, $pairs[1]); } else { debug " Adding $pairs[0] to exclude list\n"; push(@{$exclude_list->{$pairs[0]}}, "AGEN_EXCLUDE_ALL"); } } } if ($include_arg) { debug "Using include list: $include_arg"; my @list = split(/,/, $include_arg); foreach (@list) { my @pairs = split(/-/, $_); if (exists($pairs[1])) { # Remove any trailing newlines chomp($pairs[1]); debug " Adding ".$pairs[0]."->".$pairs[1]." to include list\n"; push(@{$include_list->{$pairs[0]}}, $pairs[1]); } # NOTE: it makes no sense to include all as that is the default # so ignore that scenario here, if given } } #--------------------------------------------------------------------------- # Find frameworks, components ++$step; verbose "\n$step. Searching for MCA frameworks and components\n"; my $ret; # Figure out if we're at the top level of the PRRTE tree or not. if (! (-f "VERSION" && -f "configure.ac" && -f $topdir_file)) { print("\n\nYou must run this script from the top-level directory of the PRRTE tree.\n\n"); my_exit(1); } $m4 .= "\ndnl Project names m4_define([project_name_long], [$project_name_long]) m4_define([project_name_short], [$project_name_short])\n"; # Setup MCA mca_run_global(); #--------------------------------------------------------------------------- # If we got here, all was good. Run the auto tools. ++$step; verbose "\n$step. Running autotools on top-level tree\n\n"; # Remove old versions of the files (this is probably overkill, but...) verbose "==> Remove stale files\n"; find_and_delete(qw/config.guess config.sub depcomp compile install-sh ltconfig ltmain.sh missing mkinstalldirs libtool/); # Remove the old m4 file and write the new one verbose "==> Writing m4 file with autogen.pl results\n"; unlink($m4_output_file); open(M4, ">$m4_output_file") || my_die "Can't open $m4_output_file"; print M4 $m4; close(M4); # Remove the old library path file and write the new one verbose "==> Writing txt file with all the mca component paths\n"; unlink($mca_library_paths_file); open(M4, ">$mca_library_paths_file") || my_die "Cannot open $mca_library_paths_file"; my $paths = join(":", @subdirs); print M4 $paths; close(M4); # Run autoreconf verbose "==> Running autoreconf\n"; my $cmd = "autoreconf -ivf --warnings=all,no-obsolete,no-override -I config -I config/oac"; safe_system($cmd); patch_autotools_output("."); # Per https://github.com/open-mpi/ompi/issues/8410, replace config.sub # and config.guess with known-good versions if the Autoconf-installed # versions are older. replace_config_sub_guess("."); #--------------------------------------------------------------------------- verbose " ================================================ PRRTE autogen: completed successfully. w00t! ================================================\n\n"; # Done! exit(0); prrte-4.1.1~rc2/src/0000775000175000017500000000000015203542132014426 5ustar alastairalastairprrte-4.1.1~rc2/src/hwloc/0000775000175000017500000000000015203542132015542 5ustar alastairalastairprrte-4.1.1~rc2/src/hwloc/hwloc-internal.h0000664000175000017500000003577115203542132020656 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow */ #ifndef PRTE_MCA_HWLOC_H #define PRTE_MCA_HWLOC_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #include #include #include #include #include "src/class/pmix_list.h" #include "src/class/pmix_value_array.h" BEGIN_C_DECLS /* ******************************************************************** */ /* Although we cannot bind if --without-hwloc is set, * we do still need to know some basic locality data * like on_node and not_on_node. So ensure that we * always have access to that much info by including * the definitions here, outside the if-have-hwloc test */ typedef uint16_t prte_hwloc_locality_t; #define PRTE_HWLOC_LOCALITY_T PRTE_UINT16 /** Process locality definitions */ enum { PRTE_PROC_LOCALITY_UNKNOWN = 0x0000, PRTE_PROC_NON_LOCAL = 0x8000, PRTE_PROC_ON_CLUSTER = 0x0001, PRTE_PROC_ON_CU = 0x0002, PRTE_PROC_ON_HOST = 0x0004, PRTE_PROC_ON_NODE = 0x000c, // same host PRTE_PROC_ON_PACKAGE = 0x0020, PRTE_PROC_ON_NUMA = 0x0040, PRTE_PROC_ON_L3CACHE = 0x0080, PRTE_PROC_ON_L2CACHE = 0x0100, PRTE_PROC_ON_L1CACHE = 0x0200, PRTE_PROC_ON_CORE = 0x0400, PRTE_PROC_ON_HWTHREAD = 0x0800, PRTE_PROC_ALL_LOCAL = 0x0fff, }; /** Process locality macros */ #define PRTE_PROC_ON_LOCAL_CLUSTER(n) (!!((n) &PRTE_PROC_ON_CLUSTER)) #define PRTE_PROC_ON_LOCAL_CU(n) (!!((n) &PRTE_PROC_ON_CU)) #define PRTE_PROC_ON_LOCAL_HOST(n) (!!((n) &PRTE_PROC_ON_HOST)) #define PRTE_PROC_ON_LOCAL_NODE(n) (!!((n) &PRTE_PROC_ON_LOCAL_HOST(n))) #define PRTE_PROC_ON_LOCAL_PACKAGE(n) (!!((n) &PRTE_PROC_ON_PACKAGE)) #define PRTE_PROC_ON_LOCAL_NUMA(n) (!!((n) &PRTE_PROC_ON_NUMA)) #define PRTE_PROC_ON_LOCAL_L3CACHE(n) (!!((n) &PRTE_PROC_ON_L3CACHE)) #define PRTE_PROC_ON_LOCAL_L2CACHE(n) (!!((n) &PRTE_PROC_ON_L2CACHE)) #define PRTE_PROC_ON_LOCAL_L1CACHE(n) (!!((n) &PRTE_PROC_ON_L1CACHE)) #define PRTE_PROC_ON_LOCAL_CORE(n) (!!((n) &PRTE_PROC_ON_CORE)) #define PRTE_PROC_ON_LOCAL_HWTHREAD(n) (!!((n) &PRTE_PROC_ON_HWTHREAD)) /* ******************************************************************** */ /** * Struct used to describe a section of memory (starting address * and length). This is really the same thing as an iovec, but * we include a separate type for it for at least 2 reasons: * * 1. Some OS's iovec definitions are exceedingly lame (e.g., * Solaris 9 has the length argument as an int, instead of a * size_t). * * 2. We reserve the right to expand/change this struct in the * future. */ typedef struct { /** Starting address of segment */ void *mbs_start_addr; /** Length of segment */ size_t mbs_len; } prte_hwloc_base_memory_segment_t; /** * Struct used to cache topology-level data used * for repeated lookup - the struct is attached * to the userdata of the root object of the * topology */ typedef struct { pmix_object_t super; bool computed; unsigned numa_cutoff; } prte_hwloc_topo_data_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_hwloc_topo_data_t); /** * Struct used to cache object-level data used * when computing process placement - the struct * is attached to the userdata of each object * in the topology upon first use of that object * in a placement computation */ typedef struct { pmix_object_t super; unsigned nprocs; } prte_hwloc_obj_data_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_hwloc_obj_data_t); /* define binding policies */ typedef uint16_t prte_binding_policy_t; #define PRTE_BINDING_POLICY PRTE_UINT16 /* binding directives */ #define PRTE_BIND_IF_SUPPORTED 0x1000 #define PRTE_BIND_ALLOW_OVERLOAD 0x2000 #define PRTE_BIND_GIVEN 0x4000 // overload policy was given #define PRTE_BIND_OVERLOAD_GIVEN 0x0100 /* binding policies - any changes in these * values must be reflected in prte/mca/rmaps/rmaps.h */ #define PRTE_BIND_TO_NONE 1 #define PRTE_BIND_TO_PACKAGE 2 #define PRTE_BIND_TO_NUMA 3 #define PRTE_BIND_TO_L3CACHE 4 #define PRTE_BIND_TO_L2CACHE 5 #define PRTE_BIND_TO_L1CACHE 6 #define PRTE_BIND_TO_CORE 7 #define PRTE_BIND_TO_HWTHREAD 8 #define PRTE_GET_BINDING_POLICY(pol) ((pol) &0x00ff) #define PRTE_SET_BINDING_POLICY(target, pol) \ (target) = (pol) | (((target) & 0xff00) | PRTE_BIND_GIVEN) #define PRTE_SET_DEFAULT_BINDING_POLICY(target, pol) \ do { \ if (!PRTE_BINDING_POLICY_IS_SET((target))) { \ (target) = (pol) | (((target) & 0xff00) | PRTE_BIND_IF_SUPPORTED); \ } \ } while (0); /* check if policy is set */ #define PRTE_BINDING_POLICY_IS_SET(pol) ((pol) &0x4000) /* macro to detect if binding was qualified */ #define PRTE_BINDING_REQUIRED(n) (!(PRTE_BIND_IF_SUPPORTED & (n))) /* macro to detect if binding is forced */ #define PRTE_BIND_OVERLOAD_ALLOWED(n) (PRTE_BIND_ALLOW_OVERLOAD & (n)) #define PRTE_BIND_OVERLOAD_SET(n) (PRTE_BIND_OVERLOAD_GIVEN & (n)) /* some global values */ PRTE_EXPORT extern hwloc_topology_t prte_hwloc_topology; PRTE_EXPORT extern prte_binding_policy_t prte_hwloc_default_binding_policy; PRTE_EXPORT extern hwloc_obj_type_t prte_hwloc_levels[]; PRTE_EXPORT extern char *prte_hwloc_default_cpu_list; PRTE_EXPORT extern bool prte_hwloc_default_use_hwthread_cpus; /** * Debugging output stream */ PRTE_EXPORT extern int prte_hwloc_base_output; PRTE_EXPORT extern bool prte_hwloc_base_inited; /* we always must have some minimal locality support */ #define PRTE_HWLOC_PRINT_MAX_SIZE 50 #define PRTE_HWLOC_PRINT_NUM_BUFS 16 typedef struct { char *buffers[PRTE_HWLOC_PRINT_NUM_BUFS]; int cntr; } prte_hwloc_print_buffers_t; prte_hwloc_print_buffers_t *prte_hwloc_get_print_buffer(void); extern char *prte_hwloc_print_null; PRTE_EXPORT char *prte_hwloc_base_print_locality(prte_hwloc_locality_t locality); PRTE_EXPORT extern char *prte_hwloc_base_topo_file; PRTE_EXPORT extern bool prte_hwloc_synthetic_topo; /* convenience macro for debugging */ #define PRTE_HWLOC_SHOW_BINDING(n, v, t) \ do { \ char tmp1[1024]; \ hwloc_cpuset_t bind; \ bind = prte_hwloc_alloc(); \ if (hwloc_get_cpubind(t, bind, HWLOC_CPUBIND_PROCESS) < 0) { \ pmix_output_verbose(n, v, "CANNOT DETERMINE BINDING AT %s:%d", __FILE__, __LINE__); \ } else { \ prte_hwloc_base_cset2mapstr(tmp1, sizeof(tmp1), t, bind); \ pmix_output_verbose(n, v, "BINDINGS AT %s:%d: %s", __FILE__, __LINE__, tmp1); \ } \ hwloc_bitmap_free(bind); \ } while (0); PRTE_EXPORT prte_hwloc_locality_t prte_hwloc_base_get_relative_locality(hwloc_topology_t topo, char *cpuset1, char *cpuset2); PRTE_EXPORT int prte_hwloc_base_set_default_binding(void *jdata, void *options); PRTE_EXPORT int prte_hwloc_base_set_binding_policy(void *jdata, char *spec); struct prte_rmaps_numa_node_t { pmix_list_item_t super; int index; float dist_from_closed; }; typedef struct prte_rmaps_numa_node_t prte_rmaps_numa_node_t; PMIX_CLASS_DECLARATION(prte_rmaps_numa_node_t); /** * Enum for what memory allocation policy we want for user allocations. * MAP = memory allocation policy. */ typedef enum { PRTE_HWLOC_BASE_MAP_NONE, PRTE_HWLOC_BASE_MAP_LOCAL_ONLY } prte_hwloc_base_map_t; /** * Global reflecting the MAP (set by MCA param). */ PRTE_EXPORT extern prte_hwloc_base_map_t prte_hwloc_base_map; /** * Enum for what to do if the hwloc framework tries to bind memory * and fails. BFA = bind failure action. */ typedef enum { PRTE_HWLOC_BASE_MBFA_SILENT, PRTE_HWLOC_BASE_MBFA_WARN, PRTE_HWLOC_BASE_MBFA_ERROR } prte_hwloc_base_mbfa_t; /** * Global reflecting the BFA (set by MCA param). */ PRTE_EXPORT extern prte_hwloc_base_mbfa_t prte_hwloc_base_mbfa; /** * Discover / load the hwloc topology (i.e., call hwloc_topology_init() and * hwloc_topology_load()). */ PRTE_EXPORT int prte_hwloc_base_get_topology(void); /** * Set the hwloc topology to that from the given topo file */ PRTE_EXPORT int prte_hwloc_base_set_topology(char *topofile); PRTE_EXPORT void prte_hwloc_base_setup_summary(hwloc_topology_t topo); PRTE_EXPORT hwloc_cpuset_t prte_hwloc_base_generate_cpuset(hwloc_topology_t topo, bool use_hwthread_cpus, char **cpulist); PRTE_EXPORT hwloc_cpuset_t prte_hwloc_base_filter_cpus(hwloc_topology_t topo); PRTE_EXPORT unsigned int prte_hwloc_base_get_obj_idx(hwloc_topology_t topo, hwloc_obj_t obj); PRTE_EXPORT unsigned int prte_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo, hwloc_obj_type_t target); PRTE_EXPORT hwloc_obj_t prte_hwloc_base_get_obj_by_type(hwloc_topology_t topo, hwloc_obj_type_t target, unsigned int instance); // reset all obj counters PRTE_EXPORT void prte_hwloc_base_reset_counters(void); /** * Get the number of pu's under a given hwloc object. */ PRTE_EXPORT unsigned int prte_hwloc_base_get_npus(hwloc_topology_t topo, bool use_hwthread_cpus, hwloc_cpuset_t envelope, hwloc_obj_t target); PRTE_EXPORT char *prte_hwloc_base_print_binding(prte_binding_policy_t binding); /** * Determine if there is a single cpu in a bitmap. */ PRTE_EXPORT bool prte_hwloc_base_single_cpu(hwloc_cpuset_t cpuset); /** * Provide a utility to parse a slot list against the local * cpus of given type, and produce a cpuset for the described binding */ PRTE_EXPORT int prte_hwloc_base_cpu_list_parse(const char *slot_str, hwloc_topology_t topo, bool use_hwthread_cpus, hwloc_cpuset_t cpumask); PRTE_EXPORT char *prte_hwloc_base_find_coprocessors(hwloc_topology_t topo); PRTE_EXPORT char *prte_hwloc_base_check_on_coprocessor(void); /** * Report a bind failure using the normal mechanisms if a component * fails to bind memory -- according to the value of the * hwloc_base_bind_failure_action MCA parameter. */ PRTE_EXPORT int prte_hwloc_base_report_bind_failure(const char *file, int line, const char *msg, int rc); /** * This function sets the process-wide memory affinity policy * according to prte_hwloc_base_map and prte_hwloc_base_mbfa. It needs * to be a separate, standalone function (as opposed to being done * during prte_hwloc_base_open()) because prte_hwloc_topology is not * loaded by prte_hwloc_base_open(). Hence, an upper layer needs to * invoke this function after prte_hwloc_topology has been loaded. */ PRTE_EXPORT int prte_hwloc_base_set_process_membind_policy(void); PRTE_EXPORT int prte_hwloc_base_membind(prte_hwloc_base_memory_segment_t *segs, size_t count, int node_id); PRTE_EXPORT int prte_hwloc_base_node_name_to_id(char *node_name, int *id); PRTE_EXPORT int prte_hwloc_base_memory_set(prte_hwloc_base_memory_segment_t *segments, size_t num_segments); /** * Make a prettyprint string for a hwloc_cpuset_t (e.g., "package * 2[core 3]"). */ PRTE_EXPORT char *prte_hwloc_base_cset2str(hwloc_const_cpuset_t cpuset, bool use_hwthread_cpus, bool physical, hwloc_topology_t topo); PRTE_EXPORT void prte_hwloc_get_binding_info(hwloc_const_cpuset_t cpuset, bool use_hwthread_cpus, hwloc_topology_t topo, int *pkgnum, char *cores, int sz); /* get the hwloc object that corresponds to the given processor id and type */ PRTE_EXPORT hwloc_obj_t prte_hwloc_base_get_pu(hwloc_topology_t topo, bool use_hwthread_cpus, int lid); /* get the topology "signature" so we can check for differences - caller * if responsible for freeing the returned string */ PRTE_EXPORT char *prte_hwloc_base_get_topo_signature(hwloc_topology_t topo); /* get a string describing the locality of a given process */ PRTE_EXPORT char *prte_hwloc_base_get_locality_string(hwloc_topology_t topo, char *bitmap); /* extract a location from the locality string */ PRTE_EXPORT char *prte_hwloc_base_get_location(char *locality, hwloc_obj_type_t type, unsigned index); PRTE_EXPORT prte_hwloc_locality_t prte_hwloc_compute_relative_locality(char *loc1, char *loc2); PRTE_EXPORT int prte_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen); PRTE_EXPORT int prte_hwloc_base_topology_set_flags(hwloc_topology_t topology, unsigned long flags, bool io); PRTE_EXPORT int prte_hwloc_base_open(void); PRTE_EXPORT void prte_hwloc_base_close(void); PRTE_EXPORT int prte_hwloc_base_register(void); PRTE_EXPORT int prte_hwloc_print(char **output, char *prefix, hwloc_topology_t src); PRTE_EXPORT void prte_hwloc_build_map(hwloc_topology_t topo, hwloc_cpuset_t avail, bool use_hwthread_cpus, hwloc_bitmap_t coreset); PRTE_EXPORT bool prte_hwloc_base_core_cpus(hwloc_topology_t topo); END_C_DECLS #endif /* PRTE_HWLOC_H_ */ prrte-4.1.1~rc2/src/hwloc/hwloc_base_maffinity.c0000664000175000017500000001140415203542132022062 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/include/constants.h" #include "src/hwloc/hwloc-internal.h" /* * Don't use show_help("prte", ) here (or print any error message at all). * Let the upper layer output a relevant message, because doing so may * be complicated (e.g., this might be called from the PRTE ODLS, * which has to do some extra steps to get error messages to be * displayed). */ int prte_hwloc_base_set_process_membind_policy(void) { int rc = 0, flags; hwloc_membind_policy_t policy; hwloc_cpuset_t cpuset; /* Make sure prte_hwloc_topology has been set by the time we've been called */ if (PRTE_SUCCESS != prte_hwloc_base_get_topology()) { return PRTE_ERR_BAD_PARAM; } /* Set the default memory allocation policy according to MCA param */ switch (prte_hwloc_base_map) { case PRTE_HWLOC_BASE_MAP_LOCAL_ONLY: policy = HWLOC_MEMBIND_BIND; flags = HWLOC_MEMBIND_STRICT; break; case PRTE_HWLOC_BASE_MAP_NONE: default: policy = HWLOC_MEMBIND_DEFAULT; flags = 0; break; } cpuset = hwloc_bitmap_alloc(); if (NULL == cpuset) { rc = PRTE_ERR_OUT_OF_RESOURCE; } else { int e; hwloc_get_cpubind(prte_hwloc_topology, cpuset, 0); rc = hwloc_set_membind(prte_hwloc_topology, cpuset, policy, flags); e = errno; hwloc_bitmap_free(cpuset); /* See if hwloc was able to do it. If hwloc failed due to ENOSYS, but the base_map == NONE, then it's not really an error. */ if (0 != rc && ENOSYS == e && PRTE_HWLOC_BASE_MAP_NONE == prte_hwloc_base_map) { rc = 0; } } return (0 == rc) ? PRTE_SUCCESS : PRTE_ERROR; } int prte_hwloc_base_memory_set(prte_hwloc_base_memory_segment_t *segments, size_t num_segments) { int rc = PRTE_SUCCESS; char *msg = NULL; size_t i; hwloc_cpuset_t cpuset = NULL; /* bozo check */ if (PRTE_SUCCESS != prte_hwloc_base_get_topology()) { msg = "hwloc_set_area_membind() failure - topology not available"; return prte_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc); } /* This module won't be used unless the process is already processor-bound. So find out where we're processor bound, and bind our memory there, too. */ cpuset = hwloc_bitmap_alloc(); if (NULL == cpuset) { rc = PRTE_ERR_OUT_OF_RESOURCE; msg = "hwloc_bitmap_alloc() failure"; goto out; } hwloc_get_cpubind(prte_hwloc_topology, cpuset, 0); for (i = 0; i < num_segments; ++i) { if (0 != hwloc_set_area_membind(prte_hwloc_topology, segments[i].mbs_start_addr, segments[i].mbs_len, cpuset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_STRICT)) { rc = PRTE_ERROR; msg = "hwloc_set_area_membind() failure"; goto out; } } out: if (NULL != cpuset) { hwloc_bitmap_free(cpuset); } if (PRTE_SUCCESS != rc) { return prte_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc); } return PRTE_SUCCESS; } int prte_hwloc_base_node_name_to_id(char *node_name, int *id) { /* GLB: fix me */ *id = atoi(node_name + 3); return PRTE_SUCCESS; } int prte_hwloc_base_membind(prte_hwloc_base_memory_segment_t *segs, size_t count, int node_id) { size_t i; int rc = PRTE_SUCCESS; char *msg = NULL; hwloc_cpuset_t cpuset = NULL; /* bozo check */ if (PRTE_SUCCESS != prte_hwloc_base_get_topology()) { msg = "hwloc_set_area_membind() failure - topology not available"; return prte_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc); } cpuset = hwloc_bitmap_alloc(); if (NULL == cpuset) { rc = PRTE_ERR_OUT_OF_RESOURCE; msg = "hwloc_bitmap_alloc() failure"; goto out; } hwloc_bitmap_set(cpuset, node_id); for (i = 0; i < count; i++) { if (0 != hwloc_set_area_membind(prte_hwloc_topology, segs[i].mbs_start_addr, segs[i].mbs_len, cpuset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_STRICT)) { rc = PRTE_ERROR; msg = "hwloc_set_area_membind() failure"; goto out; } } out: if (NULL != cpuset) { hwloc_bitmap_free(cpuset); } if (PRTE_SUCCESS != rc) { return prte_hwloc_base_report_bind_failure(__FILE__, __LINE__, msg, rc); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/hwloc/hwloc.c0000664000175000017500000007562115203542132017035 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/hwloc/hwloc-internal.h" #include "src/include/constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/schizo/schizo.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_tsd.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_show_help.h" #include "src/util/prte_cmd_line.h" /* * Globals */ bool prte_hwloc_base_inited = false; hwloc_topology_t prte_hwloc_topology = NULL; hwloc_cpuset_t prte_hwloc_my_cpuset = NULL; prte_hwloc_base_map_t prte_hwloc_base_map = PRTE_HWLOC_BASE_MAP_NONE; prte_hwloc_base_mbfa_t prte_hwloc_base_mbfa = PRTE_HWLOC_BASE_MBFA_WARN; prte_binding_policy_t prte_hwloc_default_binding_policy = 0; char *prte_hwloc_default_cpu_list = NULL; char *prte_hwloc_base_topo_file = NULL; int prte_hwloc_base_output = -1; bool prte_hwloc_default_use_hwthread_cpus = false; bool prte_hwloc_synthetic_topo = false; hwloc_obj_type_t prte_hwloc_levels[] = { HWLOC_OBJ_MACHINE, HWLOC_OBJ_NUMANODE, HWLOC_OBJ_PACKAGE, HWLOC_OBJ_L3CACHE, HWLOC_OBJ_L2CACHE, HWLOC_OBJ_L1CACHE, HWLOC_OBJ_CORE, HWLOC_OBJ_PU }; static char *prte_hwloc_base_binding_policy = NULL; static int verbosity = 0; static char *default_cpu_list = NULL; static bool bind_to_core = false; static bool bind_to_socket = false; static char *enum_values = NULL; int prte_hwloc_base_register(void) { int ret; char *ptr; /* debug output */ ret = pmix_mca_base_var_register("prte", "hwloc", "base", "verbose", "Debug verbosity", PMIX_MCA_BASE_VAR_TYPE_INT, &verbosity); pmix_mca_base_var_register_synonym(ret, "opal", "hwloc", "base", "verbose", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); if (0 < verbosity) { prte_hwloc_base_output = pmix_output_open(NULL); pmix_output_set_verbosity(prte_hwloc_base_output, verbosity); } /* handle some deprecated options */ prte_hwloc_default_use_hwthread_cpus = false; (void) pmix_mca_base_var_register("prte", "hwloc", "base", "use_hwthreads_as_cpus", "Use hardware threads as independent cpus", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_hwloc_default_use_hwthread_cpus); (void) pmix_mca_base_var_register("prte", "hwloc", "base", "bind_to_core", "Bind processes to cores", PMIX_MCA_BASE_VAR_TYPE_BOOL, &bind_to_core); (void) pmix_mca_base_var_register("prte", "hwloc", "base", "bind_to_socket", "Bind processes to sockets", PMIX_MCA_BASE_VAR_TYPE_BOOL, &bind_to_socket); /* hwloc_base_mbind_policy */ prte_hwloc_base_map = PRTE_HWLOC_BASE_MAP_NONE; ret = pmix_mca_base_var_register("prte", "hwloc", "default", "mem_alloc_policy", "Default general memory allocations placement policy (this is not memory binding). " "\"none\" means that no memory policy is applied. \"local_only\" means that a process' " "memory allocations will be restricted to its local NUMA domain. " "If using direct launch, this policy will not be in effect until after PMIx_Init. " "Note that operating system paging policies are unaffected by this setting. For " "example, if \"local_only\" is used and local NUMA domain memory is exhausted, a new " "memory allocation may cause paging.", PMIX_MCA_BASE_VAR_TYPE_STRING, &enum_values); if (0 > ret) { return ret; } if (NULL != enum_values) { if (0 == strncasecmp(enum_values, "none", strlen("none"))) { prte_hwloc_base_map = PRTE_HWLOC_BASE_MAP_NONE; } else if (0 == strncasecmp(enum_values, "local_only", strlen("local_only"))) { prte_hwloc_base_map = PRTE_HWLOC_BASE_MAP_LOCAL_ONLY; } else { pmix_show_help("help-prte-hwloc-base.txt", "invalid binding_policy", true, enum_values); return PRTE_ERR_SILENT; } } /* hwloc_base_bind_failure_action */ enum_values = NULL; prte_hwloc_base_mbfa = PRTE_HWLOC_BASE_MBFA_WARN; ret = pmix_mca_base_var_register("prte", "hwloc", "default", "mem_bind_failure_action", "What PRTE will do if it explicitly tries to bind memory to a specific NUMA " "location, and fails. Note that this is a different case than the general " "allocation policy described by mem_alloc_policy. A value of \"silent\" " "means that PRTE will proceed without comment. A value of \"warn\" means that " "PRTE will warn the first time this happens, but allow the job to continue " "(possibly with degraded performance). A value of \"error\" means that PRTE " "will abort the job if this happens.", PMIX_MCA_BASE_VAR_TYPE_STRING, &enum_values); if (0 > ret) { return ret; } if (NULL != enum_values) { if (0 == strncasecmp(enum_values, "silent", strlen("silent"))) { prte_hwloc_base_mbfa = PRTE_HWLOC_BASE_MBFA_SILENT; } else if (0 == strncasecmp(enum_values, "warn", strlen("warn"))) { prte_hwloc_base_mbfa = PRTE_HWLOC_BASE_MBFA_WARN; } else if (0 == strncasecmp(enum_values, "error", strlen("error"))) { prte_hwloc_base_mbfa = PRTE_HWLOC_BASE_MBFA_ERROR; } else { pmix_show_help("help-prte-hwloc-base.txt", "invalid binding_policy", true, enum_values); return PRTE_ERR_SILENT; } } /* NOTE: for future developers and readers of this code, the binding policies are strictly * limited to none, hwthread, core, l1cache, l2cache, l3cache, package, and numa * * The default binding policy can be modified by any combination of the following: * * overload-allowed - multiple processes can be bound to the same PU (core or HWT) * * if-supported - perform the binding if it is supported by the OS, but do not * generate an error if it cannot be done */ prte_hwloc_base_binding_policy = NULL; ret = pmix_mca_base_var_register("prte", NULL, NULL, "bindto", "Default policy for binding processes. Allowed values: none, hwthread, core, l1cache, " "l2cache, " "l3cache, numa, package, (\"none\" is the default when oversubscribed, \"core\" is " "the default otherwise). Allowed " "colon-delimited qualifiers: " "overload-allowed, if-supported, limit. For more details, see \"prterun --help bind-to\"" "The full directive need not be provided — " "only enough characters are required to uniquely identify the " "directive. Directive values are case insensitive", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_hwloc_base_binding_policy); (void) pmix_mca_base_var_register_synonym(ret, "prte", NULL, NULL, "bind_to", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "hwloc", "default", "binding_policy", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); if (NULL == prte_hwloc_base_binding_policy) { if (bind_to_core) { prte_hwloc_base_binding_policy = "core"; } else if (bind_to_socket) { prte_hwloc_base_binding_policy = "package"; } } /* Allow specification of a default CPU list - a comma-delimited list of cpu ranges that * are the default PUs for this DVM. CPUs are to be specified as LOGICAL indices. If a * cpuset is provided, then all process placements and bindings will be constrained to the * identified CPUs. IN ESSENCE, THIS IS A USER-DEFINED "SOFT" CGROUP. * * Example: if the default binding policy is "core", then each process will be bound to the * first unused core underneath the topological object upon which it has been mapped. In other * words, if two processes are mapped to a given package, then the first process will be bound * to core0 of that package, and the second process will be bound to core1. * * If the cpuset specified that only cores 10, 12, and 14 were to be used, then the first * process would be bound to core10 and the second process would be bound to core12. * * If the default binding policy had been set to "package", and if cores 10, 12, and 14 are all * on the same package, then both processes would be bound to cores 10, 12, and 14. Note that * they would have been bound to all PUs on the package if the cpuset had not been given. * * If cores 10 and 12 are on package0, and core14 is on package1, then if the first process is * mapped to package0 and we are using a binding policy of "package", the first process would be * bound to core10 and core12. If the second process were mapped to package1, then it would be * bound only to core14 as that is the only PU in the cpuset that lies in package1. */ default_cpu_list = NULL; ret = pmix_mca_base_var_register("prte", "hwloc", "default", "cpu_list", "Comma-separated list of ranges specifying logical cpus to be used by the DVM. " "Supported modifier:HWTCPUS (ranges specified in hwthreads) or CORECPUS " "(default: ranges specified in cores)", PMIX_MCA_BASE_VAR_TYPE_STRING, &default_cpu_list); if (NULL != default_cpu_list) { if (NULL != (ptr = strrchr(default_cpu_list, ':'))) { *ptr = '\0'; prte_hwloc_default_cpu_list = strdup(default_cpu_list); ++ptr; if (0 == strcasecmp(ptr, "HWTCPUS")) { prte_hwloc_default_use_hwthread_cpus = true; } else if (0 == strcasecmp(ptr, "CORECPUS")) { prte_hwloc_default_use_hwthread_cpus = false; } else { pmix_show_help("help-prte-hwloc-base.txt", "bad-processor-type", true, default_cpu_list, ptr); return PRTE_ERR_BAD_PARAM; } } else { prte_hwloc_default_cpu_list = strdup(default_cpu_list); } } prte_hwloc_base_topo_file = NULL; ret = pmix_mca_base_var_register("prte", "hwloc", "use", "topo_file", "Read local topology from file instead of directly sensing it", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_hwloc_base_topo_file); (void) pmix_mca_base_var_register_synonym(ret, "prte", "ras", "simulator", "topo_files", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "hwloc", "base", "use_topo_file", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); /* register parameters */ return PRTE_SUCCESS; } int prte_hwloc_base_open(void) { int rc; if (prte_hwloc_base_inited) { return PRTE_SUCCESS; } prte_hwloc_base_inited = true; /* check the provided default binding policy for correctness - specifically want to ensure * there are no disallowed qualifiers and setup the global param */ if (PRTE_SUCCESS != (rc = prte_hwloc_base_set_binding_policy(NULL, prte_hwloc_base_binding_policy))) { return rc; } return PRTE_SUCCESS; } void prte_hwloc_base_close(void) { if (!prte_hwloc_base_inited) { return; } if (NULL != prte_hwloc_default_cpu_list) { free(prte_hwloc_default_cpu_list); } /* destroy the topology */ if (NULL != prte_hwloc_topology) { hwloc_topology_destroy(prte_hwloc_topology); prte_hwloc_topology = NULL; } /* All done */ prte_hwloc_base_inited = false; } int prte_hwloc_base_set_default_binding(void *jd, void *opt) { prte_job_t *jdata = (prte_job_t*)jd; prte_rmaps_options_t *options = (prte_rmaps_options_t*)opt; prte_mapping_policy_t mpol; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16)) { /* bind to cpus */ if (options->use_hwthreads || prte_rmaps_base.require_hwtcpus) { /* if we are using hwthread cpus, then bind to those */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } else { /* bind to core */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bycore", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } } else if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL)) { /* tools are never bound */ PRTE_SET_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NONE); } else { /* if we are mapping by some object, then we default * to binding to that object */ mpol = PRTE_GET_MAPPING_POLICY(jdata->map->mapping); if (PRTE_MAPPING_BYHWTHREAD == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } else if (PRTE_MAPPING_BYCORE == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bycore", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } else if (PRTE_MAPPING_BYL1CACHE == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byL1", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L1CACHE); } else if (PRTE_MAPPING_BYL2CACHE == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byL2", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L2CACHE); } else if (PRTE_MAPPING_BYL3CACHE == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byL3", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L3CACHE); } else if (PRTE_MAPPING_BYNUMA == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bynuma", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NUMA); } else if (PRTE_MAPPING_BYPACKAGE == mpol) { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bypackage", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_PACKAGE); } else if (PRTE_MAPPING_PELIST == mpol) { if (options->use_hwthreads) { /* if we are using hwthread cpus, then bind to those */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byhwthread for pe-list", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } else { /* otherwise bind to core */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bycore for pe-list", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } } else if (PRTE_MAPPING_PPR == mpol) { if (HWLOC_OBJ_MACHINE == options->maptype) { if (options->nprocs <= 2) { /* we are mapping by node or some other non-object method */ if (options->use_hwthreads || prte_rmaps_base.require_hwtcpus) { /* if we are using hwthread cpus, then bind to those */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } else { /* otherwise bind to core */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bycore", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } } else { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bynuma", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NUMA); } } else if (HWLOC_OBJ_PACKAGE == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_PACKAGE); } else if (HWLOC_OBJ_NUMANODE== options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NUMA); } else if (HWLOC_OBJ_L1CACHE == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L1CACHE); } else if (HWLOC_OBJ_L2CACHE == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L2CACHE); } else if (HWLOC_OBJ_L3CACHE == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_L3CACHE); } else if (HWLOC_OBJ_CORE == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } else if (HWLOC_OBJ_PU == options->maptype) { PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } } else { if (options->nprocs <= 2) { /* we are mapping by node or some other non-object method */ if (options->use_hwthreads || prte_rmaps_base.require_hwtcpus) { /* if we are using hwthread cpus, then bind to those */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); } else { /* otherwise bind to core */ pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bycore", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); } } else { pmix_output_verbose(options->verbosity, options->stream, "setdefaultbinding[%d] binding not given - using bynuma", __LINE__); PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NUMA); } } } /* they might have set the overload-allowed flag while wanting PRRTE * to set the default binding - don't override it */ if (!PRTE_BIND_OVERLOAD_SET(jdata->map->binding)) { if (PRTE_BIND_OVERLOAD_ALLOWED(prte_hwloc_default_binding_policy)) { jdata->map->binding |= PRTE_BIND_ALLOW_OVERLOAD; } } return PRTE_SUCCESS; } static bool fns_init = false; static pmix_tsd_key_t print_tsd_key; char *prte_hwloc_print_null = "NULL"; static void buffer_cleanup(void *value) { int i; prte_hwloc_print_buffers_t *ptr; if (NULL != value) { ptr = (prte_hwloc_print_buffers_t *) value; for (i = 0; i < PRTE_HWLOC_PRINT_NUM_BUFS; i++) { free(ptr->buffers[i]); } free(ptr); } } prte_hwloc_print_buffers_t *prte_hwloc_get_print_buffer(void) { prte_hwloc_print_buffers_t *ptr; int ret, i; if (!fns_init) { /* setup the print_args function */ if (PRTE_SUCCESS != (ret = pmix_tsd_key_create(&print_tsd_key, buffer_cleanup))) { return NULL; } fns_init = true; } ret = pmix_tsd_getspecific(print_tsd_key, (void **) &ptr); if (PRTE_SUCCESS != ret) return NULL; if (NULL == ptr) { ptr = (prte_hwloc_print_buffers_t *) malloc(sizeof(prte_hwloc_print_buffers_t)); for (i = 0; i < PRTE_HWLOC_PRINT_NUM_BUFS; i++) { ptr->buffers[i] = (char *) malloc((PRTE_HWLOC_PRINT_MAX_SIZE + 1) * sizeof(char)); } ptr->cntr = 0; ret = pmix_tsd_setspecific(print_tsd_key, (void *) ptr); } return (prte_hwloc_print_buffers_t *) ptr; } char *prte_hwloc_base_print_locality(prte_hwloc_locality_t locality) { prte_hwloc_print_buffers_t *ptr; int idx; ptr = prte_hwloc_get_print_buffer(); if (NULL == ptr) { return prte_hwloc_print_null; } /* cycle around the ring */ if (PRTE_HWLOC_PRINT_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } idx = 0; if (PRTE_PROC_ON_LOCAL_CLUSTER(locality)) { ptr->buffers[ptr->cntr][idx++] = 'C'; ptr->buffers[ptr->cntr][idx++] = 'L'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_CU(locality)) { ptr->buffers[ptr->cntr][idx++] = 'C'; ptr->buffers[ptr->cntr][idx++] = 'U'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_NODE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'N'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_PACKAGE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'S'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_NUMA(locality)) { ptr->buffers[ptr->cntr][idx++] = 'N'; ptr->buffers[ptr->cntr][idx++] = 'M'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_L3CACHE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'L'; ptr->buffers[ptr->cntr][idx++] = '3'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_L2CACHE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'L'; ptr->buffers[ptr->cntr][idx++] = '2'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_L1CACHE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'L'; ptr->buffers[ptr->cntr][idx++] = '1'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_CORE(locality)) { ptr->buffers[ptr->cntr][idx++] = 'C'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (PRTE_PROC_ON_LOCAL_HWTHREAD(locality)) { ptr->buffers[ptr->cntr][idx++] = 'H'; ptr->buffers[ptr->cntr][idx++] = 'w'; ptr->buffers[ptr->cntr][idx++] = 't'; ptr->buffers[ptr->cntr][idx++] = ':'; } if (0 < idx) { ptr->buffers[ptr->cntr][idx - 1] = '\0'; } else if (PRTE_PROC_NON_LOCAL & locality) { ptr->buffers[ptr->cntr][idx++] = 'N'; ptr->buffers[ptr->cntr][idx++] = 'O'; ptr->buffers[ptr->cntr][idx++] = 'N'; ptr->buffers[ptr->cntr][idx++] = '\0'; } else { /* must be an unknown locality */ ptr->buffers[ptr->cntr][idx++] = 'U'; ptr->buffers[ptr->cntr][idx++] = 'N'; ptr->buffers[ptr->cntr][idx++] = 'K'; ptr->buffers[ptr->cntr][idx++] = '\0'; } return ptr->buffers[ptr->cntr]; } int prte_hwloc_base_set_binding_policy(void *jdat, char *spec) { int i; prte_binding_policy_t tmp; char **quals, *myspec, *ptr, *p2; prte_job_t *jdata = (prte_job_t *) jdat; uint16_t u16; /* set default */ tmp = 0; /* binding specification */ if (NULL == spec) { return PRTE_SUCCESS; } myspec = strdup(spec); // protect the input /* check for qualifiers */ ptr = strchr(myspec, ':'); if (NULL != ptr) { *ptr = '\0'; ++ptr; quals = PMIx_Argv_split(ptr, ':'); for (i = 0; NULL != quals[i]; i++) { if (PMIX_CHECK_CLI_OPTION(quals[i], PRTE_CLI_IF_SUPP)) { tmp |= PRTE_BIND_IF_SUPPORTED; } else if (PMIX_CHECK_CLI_OPTION(quals[i], PRTE_CLI_OVERLOAD)) { tmp |= (PRTE_BIND_ALLOW_OVERLOAD | PRTE_BIND_OVERLOAD_GIVEN); } else if (PMIX_CHECK_CLI_OPTION(quals[i], PRTE_CLI_NOOVERLOAD)) { tmp = (tmp & ~PRTE_BIND_ALLOW_OVERLOAD); tmp |= PRTE_BIND_OVERLOAD_GIVEN; } else if (PMIX_CHECK_CLI_OPTION(quals[i], PRTE_CLI_REPORT)) { if (NULL == jdata) { pmix_show_help("help-prte-rmaps-base.txt", "unsupported-default-modifier", true, "binding policy", quals[i]); free(myspec); return PRTE_ERR_SILENT; } prte_set_attribute(&jdata->attributes, PRTE_JOB_REPORT_BINDINGS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(quals[i], PRTE_CLI_LIMIT)) { if (NULL == jdata) { pmix_show_help("help-prte-rmaps-base.txt", "unsupported-default-modifier", true, "binding policy", quals[i]); free(myspec); return PRTE_ERR_SILENT; } /* Numeric value must immediately follow '=' (LIMIT=2) */ u16 = strtol(&quals[i][6], &p2, 10); if ('\0' != *p2) { /* missing the value or value is invalid */ pmix_show_help("help-prte-rmaps-base.txt", "invalid-value", true, "binding limit", "LIMIT", quals[i]); PMIx_Argv_free(quals); return PRTE_ERR_SILENT; } prte_set_attribute(&jdata->attributes, PRTE_JOB_BINDING_LIMIT, PRTE_ATTR_GLOBAL, &u16, PMIX_UINT16); } else { /* unknown option */ pmix_show_help("help-prte-hwloc-base.txt", "unrecognized-modifier", true, spec); PMIx_Argv_free(quals); free(myspec); return PRTE_ERR_BAD_PARAM; } } PMIx_Argv_free(quals); } if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_NONE)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_NONE); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_HWT)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_HWTHREAD); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_CORE)) { if (prte_rmaps_base.require_hwtcpus) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_HWTHREAD); } else { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_CORE); } } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_L1CACHE)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_L1CACHE); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_L2CACHE)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_L2CACHE); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_L3CACHE)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_L3CACHE); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_NUMA)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_NUMA); } else if (PMIX_CHECK_CLI_OPTION(myspec, PRTE_CLI_PACKAGE)) { PRTE_SET_BINDING_POLICY(tmp, PRTE_BIND_TO_PACKAGE); } else { pmix_show_help("help-prte-hwloc-base.txt", "invalid binding_policy", true, "binding", spec); free(myspec); return PRTE_ERR_BAD_PARAM; } free(myspec); if (NULL == jdata) { prte_hwloc_default_binding_policy = tmp; } else { if (NULL == jdata->map) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } jdata->map->binding = tmp; } return PRTE_SUCCESS; } static void topo_data_const(prte_hwloc_topo_data_t *ptr) { ptr->computed = false; ptr->numa_cutoff = UINT_MAX; } PMIX_CLASS_INSTANCE(prte_hwloc_topo_data_t, pmix_object_t, topo_data_const, NULL); static void obj_data_const(prte_hwloc_obj_data_t *ptr) { ptr->nprocs = 0; } PMIX_CLASS_INSTANCE(prte_hwloc_obj_data_t, pmix_object_t, obj_data_const, NULL); prrte-4.1.1~rc2/src/hwloc/help-prte-hwloc-base.txt0000664000175000017500000000504115203542132022225 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's hwloc support. # [mbind failure] PRRTE failed to bind internal memory to a specific NUMA node. This message will only be reported at most once per process. Local host: %s PID: %d File: %s:%d Message: %s Severity: %s # [invalid binding_policy] The specified %s policy is not recognized: Policy: %s Please check for a typo or ensure that the option is a supported one. # [obj-idx-failed] PRRTE failed to find a cache of a specified type. This is a highly unusual error; it may indicate a system configuration error. This additional information may be of help: Message: %s # [bad-processor-type] The default CPU list contains an invalid modifier: Default CPU list: %s Modifier: %s The modifier should indicate the type of CPU being used in the list — the only valid values are HWTCPUS (for use of hwthreads as independent CPUs) or CORECPUS (for use of cores as independent CPUs). Please correct the input. # [unrecognized-modifier] The binding request contains an unrecognized modifier: Request: %s Please check your request and try again. # [bind-upwards] Binding is performed to the first available specified object type within the object where the process was mapped. In other words, binding can only be done to the mapped object or to a resource located beneath that object. The specified binding lies above the mapping object type: Mapping level: %s Binding level: %s Please correct the map/bind directives and try again. # [pu-not-found] Construction of the binding output string failed due to inabilty to obtain a processor unit object: PU number: %u There will be no impact to your application, so we will continue but will not be able to output the binding locations. # [too-many-sites] At least one process in your application is bound to too many sites for us to report in a string. There will be no impact to your application so we will continue but will not be able to output the binding locations. # [unfound-cpu] The CPU list provided on your cmd line contains at least one ID that was not found: CPU list: %s CPU ID: %u (%s) CPU type: %s The most common reason for this is specifying the CPU list in terms of HWTs while failing to request that we treat HWTs as independent CPUs (which means we default to treating the CPU IDs as cores). Please correct your input and try again. prrte-4.1.1~rc2/src/hwloc/Makefile.am0000664000175000017500000000126615203542132017603 0ustar alastairalastair# # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2015-2020 Intel, Inc. All rights reserved. # Copyright (c) 2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # Copyright (c) 2023 Jeffrey M. Squyres. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ hwloc/help-prte-hwloc-base.txt headers += \ hwloc/hwloc-internal.h libprrte_la_SOURCES += \ hwloc/hwloc.c \ hwloc/hwloc_base_util.c \ hwloc/hwloc_base_maffinity.c prrte-4.1.1~rc2/src/hwloc/hwloc_base_util.c0000664000175000017500000022444015203542132021057 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2017 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (C) 2018 Mellanox Technologies, Ltd. * All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2019-2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #define PRTE_HWLOC_WANT_SHMEM 1 #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_ENDIAN_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #if HAVE_FCNTL_H # include #endif #include "src/include/constants.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_tsd.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/hwloc/hwloc-internal.h" bool prte_hwloc_base_core_cpus(hwloc_topology_t topo) { hwloc_obj_t obj; hwloc_obj_t pu; obj = hwloc_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0); if (NULL == obj) { return false; } /* see if the cpuset of a core match that of a PU */ pu = hwloc_get_obj_by_type(topo, HWLOC_OBJ_PU, 0); /* if the two are equal, then we really don't have * cores in this topology */ if (hwloc_bitmap_isequal(obj->cpuset, pu->cpuset)) { return false; } /* we do have cores */ return true; } /* * Provide the hwloc object that corresponds to the given * processor id of the given type. Remember: "processor" here [usually] means "core" -- * except that on some platforms, hwloc won't find any cores; it'll * only find PUs (!). On such platforms, then do the same calculation * but with PUs instead of COREs. */ hwloc_obj_t prte_hwloc_base_get_pu(hwloc_topology_t topo, bool use_hwthread_cpus, int lid) { hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE; hwloc_obj_t obj; /* hwloc isn't able to find cores on all platforms. Example: PPC64 running RHEL 5.4 (linux kernel 2.6.18) only reports NUMA nodes and PU's. Fine. However, note that hwloc_get_obj_by_type() will return NULL in 2 (effectively) different cases: - no objects of the requested type were found - the Nth object of the requested type was not found So first we have to see if we can find *any* cores by looking for the 0th core. If we find it, then try to find the Nth core. Otherwise, try to find the Nth PU. */ if (use_hwthread_cpus || !prte_hwloc_base_core_cpus(topo)) { obj_type = HWLOC_OBJ_PU; } pmix_output_verbose(5, prte_hwloc_base_output, "Searching for %d LOGICAL PU", lid); /* Now do the actual lookup. */ obj = hwloc_get_obj_by_type(topo, obj_type, lid); pmix_output_verbose(5, prte_hwloc_base_output, "logical cpu %d %s found", lid, (NULL == obj) ? "not" : "is"); /* Found the right core (or PU). Return the object */ return obj; } hwloc_cpuset_t prte_hwloc_base_generate_cpuset(hwloc_topology_t topo, bool use_hwthread_cpus, char **cpulist) { hwloc_cpuset_t avail = NULL, pucpus, res; char **ranges = NULL, **range = NULL; int idx, cpu, start, end; hwloc_obj_t pu; char **cache = NULL; char tmp[256]; /* find the specified logical cpus */ ranges = PMIx_Argv_split(*cpulist, ','); avail = hwloc_bitmap_alloc(); hwloc_bitmap_zero(avail); res = hwloc_bitmap_alloc(); pucpus = hwloc_bitmap_alloc(); for (idx = 0; idx < PMIx_Argv_count(ranges); idx++) { range = PMIx_Argv_split(ranges[idx], '-'); switch (PMIx_Argv_count(range)) { case 1: /* only one cpu given - get that object */ cpu = strtoul(range[0], NULL, 10); pu = prte_hwloc_base_get_pu(topo, use_hwthread_cpus, cpu); if (NULL == pu) { pmix_show_help("help-prte-hwloc-base.txt", "unfound-cpu", true, *cpulist, cpu, "logical", use_hwthread_cpus ? "hwthread" : "core"); PMIx_Argv_free(ranges); PMIx_Argv_free(range); PMIx_Argv_free(cache); hwloc_bitmap_free(avail); hwloc_bitmap_free(res); hwloc_bitmap_free(pucpus); return NULL; } hwloc_bitmap_and(pucpus, pu->cpuset, hwloc_topology_get_allowed_cpuset(topo)); hwloc_bitmap_or(res, avail, pucpus); hwloc_bitmap_copy(avail, res); // cache the cpu PMIx_Argv_append_nosize(&cache, range[0]); break; case 2: /* range given */ start = strtoul(range[0], NULL, 10); end = strtoul(range[1], NULL, 10); for (cpu = start; cpu <= end; cpu++) { pu = prte_hwloc_base_get_pu(topo, use_hwthread_cpus, cpu); if (NULL == pu) { pmix_show_help("help-prte-hwloc-base.txt", "unfound-cpu", true, cpulist, cpu, "logical", use_hwthread_cpus ? "hwthread" : "core"); PMIx_Argv_free(ranges); PMIx_Argv_free(range); PMIx_Argv_free(cache); hwloc_bitmap_free(avail); hwloc_bitmap_free(res); hwloc_bitmap_free(pucpus); return NULL; } hwloc_bitmap_and(pucpus, pu->cpuset, hwloc_topology_get_allowed_cpuset(topo)); hwloc_bitmap_or(res, avail, pucpus); hwloc_bitmap_copy(avail, res); // cache the cpu snprintf(tmp, 256, "%d", cpu); PMIx_Argv_append_nosize(&cache, tmp); } break; default: break; } PMIx_Argv_free(range); } if (NULL != ranges) { PMIx_Argv_free(ranges); } hwloc_bitmap_free(res); hwloc_bitmap_free(pucpus); // update the cpulist in case it was expanded free(*cpulist); *cpulist = PMIx_Argv_join(cache, ','); PMIx_Argv_free(cache); return avail; } void prte_hwloc_base_setup_summary(hwloc_topology_t topo) { hwloc_obj_t root; prte_hwloc_topo_data_t *sum; int val; unsigned width, w, m, N, last; hwloc_bitmap_t *numas; hwloc_obj_t obj; /* Historically, CPU packages contained a single cpu die * and nothing else. NUMA was therefore determined by simply * looking at the memory bus attached to the socket where * the package resided - all cpus in the package were * exclusively "under" that NUMA. Since each socket had a * unique NUMA, you could easily map by them. * More recently, packages have started to contain multiple * cpu dies as well as memory and sometimes even fabric die. * In these cases, the memory bus of the cpu dies in the * package generally share any included memory die. This * complicates the memory situation, leaving NUMA domains * no longer cleanly delineated by processor (i.e.., the * NUMA domains overlap each other). * * Fortunately, the OS index of non-CPU NUMA domains starts * at 255 and counts downward (at least for GPUs) - while * the index of CPU NUMA domains starts at 0 and counts * upward. We can therefore separate the two by excluding * NUMA domains with an OS index above the level where * they first begin to intersect */ root = hwloc_get_root_obj(topo); if (NULL == root->userdata) { root->userdata = (void *) PMIX_NEW(prte_hwloc_topo_data_t); } sum = (prte_hwloc_topo_data_t *) root->userdata; /* only need to do this once */ if (sum->computed) { return; } sum->computed = true; /* compute the CPU NUMA cutoff for this topology */ val = hwloc_get_nbobjs_by_type(topo, HWLOC_OBJ_NUMANODE); if (0 > val) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return; } width = val; if (0 == width) { sum->numa_cutoff = 0; return; } numas = (hwloc_bitmap_t*)malloc(width * sizeof(hwloc_bitmap_t)); N = 0; last = 0; for (w=0; w < UINT_MAX && N < width; w++) { /* get the object at this index */ obj = hwloc_get_numanode_obj_by_os_index(topo, w); if (NULL == obj) { continue; } /* check for overlap with all preceding numas */ for (m=0; m < N; m++) { if (hwloc_bitmap_intersects(obj->cpuset, numas[m])) { // if it intersects anyone, then we are done sum->numa_cutoff = last+1; break; } } if (UINT_MAX != sum->numa_cutoff) { break; } else { last = w; /* cache this bitmap */ numas[N] = hwloc_bitmap_alloc(); hwloc_bitmap_copy(numas[N], obj->cpuset); ++N; } } if (UINT_MAX == sum->numa_cutoff) { sum->numa_cutoff = last + 1; } for (m=0; m < N; m++) { hwloc_bitmap_free(numas[m]); } free(numas); } /* determine the node-level available cpuset based on * online vs allowed vs user-specified cpus */ hwloc_cpuset_t prte_hwloc_base_filter_cpus(hwloc_topology_t topo) { hwloc_cpuset_t avail = NULL; /* process any specified default cpu set against this topology */ if (NULL == prte_hwloc_default_cpu_list) { PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base: no cpus specified - using root available cpuset")); avail = hwloc_bitmap_alloc(); hwloc_bitmap_copy(avail, hwloc_topology_get_allowed_cpuset(topo)); } else { PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base: filtering cpuset")); avail = prte_hwloc_base_generate_cpuset(topo, prte_hwloc_default_use_hwthread_cpus, &prte_hwloc_default_cpu_list); } return avail; } static void fill_cache_line_size(void) { bool found = false; unsigned size = 0, cache_level = 2, i = 0; hwloc_obj_type_t cache_object = HWLOC_OBJ_L2CACHE; hwloc_obj_t obj; /* Look for the smallest L2 cache size */ size = 4096; while (cache_level > 0 && !found) { i = 0; while (1) { obj = hwloc_get_obj_by_type(prte_hwloc_topology, cache_object, i); if (NULL == obj) { --cache_level; cache_object = HWLOC_OBJ_L1CACHE; break; } else { if (NULL != obj->attr && obj->attr->cache.linesize > 0 && size > obj->attr->cache.linesize) { size = obj->attr->cache.linesize; found = true; } } ++i; } } /* If we found an L2 cache size in the hwloc data, save it in prte_cache_line_size. Otherwise, we'll leave whatever default was set in prte_init.c */ if (found) { prte_cache_line_size = (int) size; } } int prte_hwloc_base_get_topology(void) { int rc; unsigned i, j; hwloc_obj_t obj; pmix_output_verbose(2, prte_hwloc_base_output, "hwloc:base:get_topology"); /* see if we already have it */ if (NULL != prte_hwloc_topology) { return PRTE_SUCCESS; } if (NULL == prte_hwloc_base_topo_file) { pmix_output_verbose(1, prte_hwloc_base_output, "hwloc:base discovering topology"); if (0 != hwloc_topology_init(&prte_hwloc_topology) || 0 != prte_hwloc_base_topology_set_flags(prte_hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM, true) || 0 != hwloc_topology_load(prte_hwloc_topology)) { PRTE_ERROR_LOG(PRTE_ERR_NOT_SUPPORTED); return PRTE_ERR_NOT_SUPPORTED; } } else { pmix_output_verbose(1, prte_hwloc_base_output, "hwloc:base loading topology from file %s", prte_hwloc_base_topo_file); if (PRTE_SUCCESS != (rc = prte_hwloc_base_set_topology(prte_hwloc_base_topo_file))) { return rc; } prte_hwloc_synthetic_topo = true; } /* remove a few things that we know cause topology differences * that aren't really of interest */ obj = hwloc_get_root_obj(prte_hwloc_topology); for (i = 0; i < obj->infos_count; i++) { if (NULL == obj->infos[i].name || NULL == obj->infos[i].value) { continue; } if (0 == strncmp(obj->infos[i].name, "HostName", strlen("HostName")) || 0 == strncmp(obj->infos[i].name, "ProcessName", strlen("ProcessName"))) { free(obj->infos[i].name); free(obj->infos[i].value); /* left justify the array */ for (j = i; j < obj->infos_count - 1; j++) { obj->infos[j] = obj->infos[j + 1]; } obj->infos[obj->infos_count - 1].name = NULL; obj->infos[obj->infos_count - 1].value = NULL; obj->infos_count--; } } /* fill prte_cache_line_size global with the smallest L1 cache line size */ fill_cache_line_size(); // create the summary prte_hwloc_base_setup_summary(prte_hwloc_topology); return PRTE_SUCCESS; } int prte_hwloc_base_set_topology(char *topofile) { int rc; PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:set_topology %s", topofile)); if (NULL != prte_hwloc_topology) { hwloc_topology_destroy(prte_hwloc_topology); } if (0 != hwloc_topology_init(&prte_hwloc_topology)) { return PRTE_ERR_NOT_SUPPORTED; } if (0 != hwloc_topology_set_xml(prte_hwloc_topology, topofile)) { hwloc_topology_destroy(prte_hwloc_topology); PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:set_topology bad topo file")); return PRTE_ERR_NOT_SUPPORTED; } /* since we are loading this from an external source, we have to * explicitly set a flag so hwloc sets things up correctly */ rc = prte_hwloc_base_topology_set_flags(prte_hwloc_topology, 0, true); if (0 != rc) { hwloc_topology_destroy(prte_hwloc_topology); return PRTE_ERR_NOT_SUPPORTED; } if (0 != hwloc_topology_load(prte_hwloc_topology)) { hwloc_topology_destroy(prte_hwloc_topology); PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:set_topology failed to load")); return PRTE_ERR_NOT_SUPPORTED; } /* all done */ return PRTE_SUCCESS; } int prte_hwloc_base_report_bind_failure(const char *file, int line, const char *msg, int rc) { static int already_reported = 0; if (!already_reported && PRTE_HWLOC_BASE_MBFA_SILENT != prte_hwloc_base_mbfa) { pmix_show_help( "help-prte-hwloc-base.txt", "mbind failure", true, prte_process_info.nodename, getpid(), file, line, msg, (PRTE_HWLOC_BASE_MBFA_WARN == prte_hwloc_base_mbfa) ? "Warning -- your job will continue, but possibly with degraded performance" : "ERROR -- your job may abort or behave erraticly"); already_reported = 1; return rc; } return PRTE_SUCCESS; } /* determine if there is a single cpu in a bitmap */ bool prte_hwloc_base_single_cpu(hwloc_cpuset_t cpuset) { int i; bool one = false; /* count the number of bits that are set - there is * one bit for each available pu. We could just * subtract the first and last indices, but there * may be "holes" in the bitmap corresponding to * offline or unallowed cpus - so we have to * search for them. Return false if we anything * other than one */ for (i = hwloc_bitmap_first(cpuset); i <= hwloc_bitmap_last(cpuset); i++) { if (hwloc_bitmap_isset(cpuset, i)) { if (one) { return false; } one = true; } } return one; } /* get the number of pu's under a given hwloc object */ unsigned int prte_hwloc_base_get_npus(hwloc_topology_t topo, bool use_hwthread_cpus, hwloc_cpuset_t envelope, hwloc_obj_t obj) { unsigned int cnt = 0; hwloc_cpuset_t avail; if (NULL == obj->cpuset) { return 0; } if (NULL == envelope) { avail = hwloc_bitmap_dup(obj->cpuset); } else { avail = hwloc_bitmap_alloc(); hwloc_bitmap_and(avail, obj->cpuset, envelope); } if (!use_hwthread_cpus) { /* if we are treating cores as cpus, then we really * want to know how many cores are in this object. * hwloc sets a bit for each "pu", so we can't just * count bits in this case as there may be more than * one hwthread/core. Instead, find the number of cores * in the system */ cnt = hwloc_get_nbobjs_inside_cpuset_by_type(topo, avail, HWLOC_OBJ_CORE); } else { /* count the number of bits that are set - there is * one bit for each available pu. We could just * subtract the first and last indices, but there * may be "holes" in the bitmap corresponding to * offline or unallowed cpus - so we count them with * the bitmap "weight" (a.k.a. population count) function */ cnt = hwloc_bitmap_weight(avail); } hwloc_bitmap_free(avail); return cnt; } unsigned int prte_hwloc_base_get_obj_idx(hwloc_topology_t topo, hwloc_obj_t obj) { hwloc_obj_t ptr; unsigned int nobjs, i; PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:get_idx")); nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, obj->type); PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:get_idx found %u objects of type %s", nobjs, hwloc_obj_type_string(obj->type))); /* find this object */ for (i = 0; i < nobjs; i++) { ptr = prte_hwloc_base_get_obj_by_type(topo, obj->type, i); if (ptr == obj) { return i; } } /* if we get here, it wasn't found */ pmix_show_help("help-prte-hwloc-base.txt", "obj-idx-failed", true, hwloc_obj_type_string(obj->type)); return UINT_MAX; } unsigned int prte_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo, hwloc_obj_type_t target) { unsigned w, rc; hwloc_obj_t obj, root; prte_hwloc_topo_data_t *sum; /* if the type is NUMA, then we need to only count the * CPU NUMAs and ignore the GPU NUMAs as we only deal * with CPUs at this time */ if (HWLOC_OBJ_NUMANODE == target) { root = hwloc_get_root_obj(topo); sum = (prte_hwloc_topo_data_t *) root->userdata; if (NULL == sum) { return 0; } rc = 0; for (w=0; w < sum->numa_cutoff; w++) { obj = hwloc_get_numanode_obj_by_os_index(topo, w); if (NULL != obj) { ++rc; } } return rc; } rc = hwloc_get_nbobjs_by_type(topo, target); if (UINT_MAX == rc) { pmix_output(0, "UNKNOWN HWLOC ERROR"); return 0; } return rc; } hwloc_obj_t prte_hwloc_base_get_obj_by_type(hwloc_topology_t topo, hwloc_obj_type_t target, unsigned int instance) { unsigned w, cnt; hwloc_obj_t obj, root; prte_hwloc_topo_data_t *sum; /* if we are looking for NUMA, then ignore all the * GPU NUMAs */ if (HWLOC_OBJ_NUMANODE == target) { root = hwloc_get_root_obj(topo); sum = (prte_hwloc_topo_data_t *) root->userdata; if (NULL == sum) { return NULL; } cnt = 0; for (w=0; w < sum->numa_cutoff; w++) { obj = hwloc_get_numanode_obj_by_os_index(topo, w); if (NULL != obj) { if (cnt == instance) { return obj; } ++cnt; } } return NULL; } return hwloc_get_obj_by_type(topo, target, instance); } /* The current slot_list notation only goes to the core level - i.e., the location * is specified as package:core. Thus, the code below assumes that all locations * are to be parsed under that notation. */ static int package_to_cpu_set(char *cpus, hwloc_topology_t topo, hwloc_bitmap_t cpumask) { char **range; int range_cnt; int lower_range, upper_range; int package_id; hwloc_obj_t obj; if ('*' == cpus[0]) { /* requesting cpumask for ALL packages */ obj = hwloc_get_root_obj(topo); /* set to all available processors - essentially, * this specification equates to unbound */ hwloc_bitmap_or(cpumask, cpumask, obj->cpuset); return PRTE_SUCCESS; } range = PMIx_Argv_split(cpus, '-'); range_cnt = PMIx_Argv_count(range); switch (range_cnt) { case 1: /* no range was present, so just one package given */ package_id = atoi(range[0]); obj = prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_PACKAGE, package_id); /* get the available cpus for this package */ hwloc_bitmap_or(cpumask, cpumask, obj->cpuset); break; case 2: /* range of packages was given */ lower_range = atoi(range[0]); upper_range = atoi(range[1]); /* cycle across the range of packages */ for (package_id = lower_range; package_id <= upper_range; package_id++) { obj = prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_PACKAGE, package_id); /* set the available cpus for this package bits in the bitmask */ hwloc_bitmap_or(cpumask, cpumask, obj->cpuset); } break; default: PMIx_Argv_free(range); return PRTE_ERROR; } PMIx_Argv_free(range); return PRTE_SUCCESS; } static int package_core_to_cpu_set(char *package_core_list, hwloc_topology_t topo, hwloc_bitmap_t cpumask) { int rc = PRTE_SUCCESS, i, j; char **package_core, *corestr; char **range, **list; int range_cnt; int lower_range, upper_range; int package_id, core_id; hwloc_obj_t package, core; hwloc_obj_type_t obj_type = HWLOC_OBJ_CORE; unsigned int npus; bool hwthreadcpus = false; package_core = PMIx_Argv_split(package_core_list, ':'); package_id = atoi(package_core[0]); /* get the object for this package id */ package = prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_PACKAGE, package_id); if (NULL == package) { PMIx_Argv_free(package_core); return PRTE_ERR_NOT_FOUND; } /* as described in comment near top of file, hwloc isn't able * to find cores on all platforms. Adjust the type here if * required */ if (NULL == prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_CORE, 0)) { obj_type = HWLOC_OBJ_PU; hwthreadcpus = true; } npus = prte_hwloc_base_get_npus(topo, hwthreadcpus, NULL, package); npus = npus * package_id; for (i = 1; NULL != package_core[i]; i++) { if ('C' == package_core[i][0] || 'c' == package_core[i][0]) { corestr = &package_core[i][1]; } else { corestr = package_core[i]; } if ('*' == corestr[0]) { /* set to all cpus on this package */ hwloc_bitmap_or(cpumask, cpumask, package->cpuset); /* we are done - already assigned all cores! */ rc = PRTE_SUCCESS; break; } else { range = PMIx_Argv_split(corestr, '-'); range_cnt = PMIx_Argv_count(range); /* see if a range was set or not */ switch (range_cnt) { case 1: /* only one core, or a list of cores, specified */ list = PMIx_Argv_split(range[0], ','); for (j = 0; NULL != list[j]; j++) { /* get the indexed core from this package */ core_id = atoi(list[j]) + npus; /* get that object */ core = prte_hwloc_base_get_obj_by_type(topo, obj_type, core_id); if (NULL == core) { rc = PRTE_ERR_NOT_FOUND; break; } /* get the cpus */ hwloc_bitmap_or(cpumask, cpumask, core->cpuset); } PMIx_Argv_free(list); break; case 2: /* range of core id's was given */ pmix_output_verbose(5, prte_hwloc_base_output, "range of cores given: start %s stop %s", range[0], range[1]); lower_range = atoi(range[0]); upper_range = atoi(range[1]); for (j = lower_range; j <= upper_range; j++) { /* get the indexed core from this package */ core_id = j + npus; /* get that object */ core = prte_hwloc_base_get_obj_by_type(topo, obj_type, core_id); if (NULL == core) { rc = PRTE_ERR_NOT_FOUND; break; } /* get the cpus add them into the result */ hwloc_bitmap_or(cpumask, cpumask, core->cpuset); } break; default: PMIx_Argv_free(range); PMIx_Argv_free(package_core); return PRTE_ERROR; } PMIx_Argv_free(range); } } PMIx_Argv_free(package_core); return rc; } int prte_hwloc_base_cpu_list_parse(const char *slot_str, hwloc_topology_t topo, bool use_hwthread_cpus, hwloc_cpuset_t cpumask) { char **item, **rngs, *lst; int rc, i, j, k; hwloc_obj_t pu; char **range, **list; size_t range_cnt; int core_id, lower_range, upper_range; /* bozo checks */ if (NULL == prte_hwloc_topology) { return PRTE_ERR_NOT_SUPPORTED; } if (NULL == slot_str || 0 == strlen(slot_str)) { return PRTE_ERR_BAD_PARAM; } pmix_output_verbose(5, prte_hwloc_base_output, "slot assignment: slot_list == %s", slot_str); /* split at ';' */ item = PMIx_Argv_split(slot_str, ';'); /* start with a clean mask */ hwloc_bitmap_zero(cpumask); /* loop across the items and accumulate the mask */ for (i = 0; NULL != item[i]; i++) { pmix_output_verbose(5, prte_hwloc_base_output, "working assignment %s", item[i]); /* if they specified "package" by starting with an S/s, * or if they use package:core notation, then parse the * package/core info */ if ('P' == item[i][0] || 'p' == item[i][0] || 'S' == item[i][0] || 's' == item[i][0] || // backward compatibility NULL != strchr(item[i], ':')) { /* specified a package */ if (NULL == strchr(item[i], ':')) { /* binding just to the package level, though * it could specify multiple packages * Skip the P and look for ranges */ rngs = PMIx_Argv_split(&item[i][1], ','); for (j = 0; NULL != rngs[j]; j++) { if (PRTE_SUCCESS != (rc = package_to_cpu_set(rngs[j], topo, cpumask))) { PMIx_Argv_free(rngs); PMIx_Argv_free(item); return rc; } } PMIx_Argv_free(rngs); } else { if ('P' == item[i][0] || 'p' == item[i][0] || 'S' == item[i][0] || 's' == item[i][0]) { lst = &item[i][1]; } else { lst = item[i]; } if (PRTE_SUCCESS != (rc = package_core_to_cpu_set(lst, topo, cpumask))) { PMIx_Argv_free(item); return rc; } } } else { rngs = PMIx_Argv_split(item[i], ','); for (k = 0; NULL != rngs[k]; k++) { /* just a core specification - see if one or a range was given */ range = PMIx_Argv_split(rngs[k], '-'); range_cnt = PMIx_Argv_count(range); /* see if a range was set or not */ switch (range_cnt) { case 1: /* only one core, or a list of cores, specified */ list = PMIx_Argv_split(range[0], ','); for (j = 0; NULL != list[j]; j++) { core_id = atoi(list[j]); /* find the specified available cpu */ if (NULL == (pu = prte_hwloc_base_get_pu(topo, use_hwthread_cpus, core_id))) { PMIx_Argv_free(range); PMIx_Argv_free(item); PMIx_Argv_free(rngs); PMIx_Argv_free(list); return PRTE_ERR_NOT_FOUND; } /* get the cpus for that object and set them in the massk*/ hwloc_bitmap_or(cpumask, cpumask, pu->cpuset); } PMIx_Argv_free(list); break; case 2: /* range of core id's was given */ lower_range = atoi(range[0]); upper_range = atoi(range[1]); for (core_id = lower_range; core_id <= upper_range; core_id++) { /* find the specified logical available cpu */ if (NULL == (pu = prte_hwloc_base_get_pu(topo, use_hwthread_cpus, core_id))) { PMIx_Argv_free(range); PMIx_Argv_free(item); PMIx_Argv_free(rngs); return PRTE_ERR_NOT_FOUND; } /* get the cpus for that object and set them in the mask*/ hwloc_bitmap_or(cpumask, cpumask, pu->cpuset); } break; default: PMIx_Argv_free(range); PMIx_Argv_free(item); PMIx_Argv_free(rngs); return PRTE_ERROR; } PMIx_Argv_free(range); } PMIx_Argv_free(rngs); } } PMIx_Argv_free(item); return PRTE_SUCCESS; } static void prte_hwloc_base_get_relative_locality_by_depth(hwloc_topology_t topo, unsigned d, hwloc_cpuset_t loc1, hwloc_cpuset_t loc2, prte_hwloc_locality_t *locality, bool *shared) { unsigned width, w; hwloc_obj_t obj; int sect1, sect2; /* get the width of the topology at this depth */ width = hwloc_get_nbobjs_by_depth(topo, d); /* scan all objects at this depth to see if * our locations overlap with them */ for (w = 0; w < width; w++) { /* get the object at this depth/index */ obj = hwloc_get_obj_by_depth(topo, d, w); /* see if our locations intersect with the cpuset for this obj */ sect1 = hwloc_bitmap_intersects(obj->cpuset, loc1); sect2 = hwloc_bitmap_intersects(obj->cpuset, loc2); /* if both intersect, then we share this level */ if (sect1 && sect2) { *shared = true; switch (obj->type) { case HWLOC_OBJ_PACKAGE: *locality |= PRTE_PROC_ON_PACKAGE; break; case HWLOC_OBJ_NUMANODE: *locality |= PRTE_PROC_ON_NUMA; break; case HWLOC_OBJ_L3CACHE: *locality |= PRTE_PROC_ON_L3CACHE; break; case HWLOC_OBJ_L2CACHE: *locality |= PRTE_PROC_ON_L2CACHE; break; case HWLOC_OBJ_L1CACHE: *locality |= PRTE_PROC_ON_L1CACHE; break; case HWLOC_OBJ_CORE: *locality |= PRTE_PROC_ON_CORE; break; case HWLOC_OBJ_PU: *locality |= PRTE_PROC_ON_HWTHREAD; break; default: /* just ignore it */ break; } break; } /* otherwise, we don't share this * object - but we still might share another object * on this level, so we have to keep searching */ } } prte_hwloc_locality_t prte_hwloc_base_get_relative_locality(hwloc_topology_t topo, char *cpuset1, char *cpuset2) { prte_hwloc_locality_t locality; hwloc_cpuset_t loc1, loc2; unsigned depth, d; bool shared; hwloc_obj_type_t type; /* start with what we know - they share a node on a cluster * NOTE: we may alter that latter part as hwloc's ability to * sense multi-cu, multi-cluster systems grows */ locality = PRTE_PROC_ON_NODE | PRTE_PROC_ON_HOST | PRTE_PROC_ON_CU | PRTE_PROC_ON_CLUSTER; /* if either cpuset is NULL, then that isn't bound */ if (NULL == cpuset1 || NULL == cpuset2) { return locality; } /* get the max depth of the topology */ depth = hwloc_topology_get_depth(topo); /* convert the strings to cpusets */ loc1 = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(loc1, cpuset1); loc2 = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(loc2, cpuset2); /* start at the first depth below the top machine level */ for (d = 1; d < depth; d++) { shared = false; /* get the object type at this depth */ type = hwloc_get_depth_type(topo, d); /* if it isn't one of interest, then ignore it */ if (HWLOC_OBJ_NUMANODE != type && HWLOC_OBJ_PACKAGE != type && HWLOC_OBJ_L3CACHE != type && HWLOC_OBJ_L2CACHE != type && HWLOC_OBJ_L1CACHE != type && HWLOC_OBJ_CORE != type && HWLOC_OBJ_PU != type) { continue; } prte_hwloc_base_get_relative_locality_by_depth(topo, d, loc1, loc2, &locality, &shared); /* if we spanned the entire width without finding * a point of intersection, then no need to go * deeper */ if (!shared) { break; } } prte_hwloc_base_get_relative_locality_by_depth(topo, (unsigned) HWLOC_TYPE_DEPTH_NUMANODE, loc1, loc2, &locality, &shared); pmix_output_verbose(5, prte_hwloc_base_output, "locality: %s", prte_hwloc_base_print_locality(locality)); hwloc_bitmap_free(loc1); hwloc_bitmap_free(loc2); return locality; } /* searches the given topology for coprocessor objects and returns * their serial numbers as a comma-delimited string, or NULL * if no coprocessors are found */ char *prte_hwloc_base_find_coprocessors(hwloc_topology_t topo) { hwloc_obj_t osdev; unsigned i; char **cps = NULL; char *cpstring = NULL; int depth; /* coprocessors are recorded under OS_DEVICEs, so first * see if we have any of those */ if (HWLOC_TYPE_DEPTH_UNKNOWN == (depth = hwloc_get_type_depth(topo, HWLOC_OBJ_OS_DEVICE))) { PMIX_OUTPUT_VERBOSE( (5, prte_hwloc_base_output, "hwloc:base:find_coprocessors: NONE FOUND IN TOPO")); return NULL; } /* check the device objects for coprocessors */ osdev = hwloc_get_obj_by_depth(topo, depth, 0); while (NULL != osdev) { if (HWLOC_OBJ_OSDEV_COPROC == osdev->attr->osdev.type) { /* got one! find and save its serial number */ for (i = 0; i < osdev->infos_count; i++) { if (0 == strncmp(osdev->infos[i].name, "MICSerialNumber", strlen("MICSerialNumber"))) { PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:find_coprocessors: coprocessor %s found", osdev->infos[i].value)); PMIx_Argv_append_nosize(&cps, osdev->infos[i].value); } } } osdev = osdev->next_cousin; } if (NULL != cps) { cpstring = PMIx_Argv_join(cps, ','); PMIx_Argv_free(cps); } PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:find_coprocessors: hosting coprocessors %s", (NULL == cpstring) ? "NONE" : cpstring)); return cpstring; } #define PRTE_HWLOC_MAX_ELOG_LINE 1024 static char *hwloc_getline(FILE *fp) { char *ret, *buff; char input[PRTE_HWLOC_MAX_ELOG_LINE]; ret = fgets(input, PRTE_HWLOC_MAX_ELOG_LINE, fp); if (NULL != ret) { input[strlen(input) - 1] = '\0'; /* remove newline */ buff = strdup(input); return buff; } return NULL; } /* checks local environment to determine if this process * is on a coprocessor - if so, it returns the serial number * as a string, or NULL if it isn't on a coprocessor */ char *prte_hwloc_base_check_on_coprocessor(void) { /* this support currently is limited to Intel Phi processors * but will hopefully be extended as we get better, more * generalized ways of identifying coprocessors */ FILE *fp; char *t, *cptr, *e, *cp = NULL; if (NULL == (fp = fopen("/proc/elog", "r"))) { /* nothing we can do */ return NULL; } /* look for the line containing the serial number of this * card - usually the first line in the file */ while (NULL != (cptr = hwloc_getline(fp))) { if (NULL != (t = strstr(cptr, "Card"))) { /* we want the string right after this - delimited by * a colon at the end */ t += 5; // move past "Card " if (NULL == (e = strchr(t, ':'))) { /* not what we were expecting */ free(cptr); continue; } *e = '\0'; cp = strdup(t); free(cptr); break; } free(cptr); } fclose(fp); PMIX_OUTPUT_VERBOSE((5, prte_hwloc_base_output, "hwloc:base:check_coprocessor: on coprocessor %s", (NULL == cp) ? "NONE" : cp)); return cp; } char *prte_hwloc_base_print_binding(prte_binding_policy_t binding) { char *ret, *bind; prte_hwloc_print_buffers_t *ptr; switch (PRTE_GET_BINDING_POLICY(binding)) { case PRTE_BIND_TO_NONE: bind = "NONE"; break; case PRTE_BIND_TO_PACKAGE: bind = "PACKAGE"; break; case PRTE_BIND_TO_NUMA: bind = "NUMA"; break; case PRTE_BIND_TO_L3CACHE: bind = "L3CACHE"; break; case PRTE_BIND_TO_L2CACHE: bind = "L2CACHE"; break; case PRTE_BIND_TO_L1CACHE: bind = "L1CACHE"; break; case PRTE_BIND_TO_CORE: bind = "CORE"; break; case PRTE_BIND_TO_HWTHREAD: bind = "HWTHREAD"; break; default: bind = "UNKNOWN"; } ptr = prte_hwloc_get_print_buffer(); if (NULL == ptr) { return prte_hwloc_print_null; } /* cycle around the ring */ if (PRTE_HWLOC_PRINT_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } if (!PRTE_BINDING_REQUIRED(binding) && PRTE_BIND_OVERLOAD_ALLOWED(binding)) { snprintf(ptr->buffers[ptr->cntr], PRTE_HWLOC_PRINT_MAX_SIZE, "%s:IF-SUPPORTED:OVERLOAD-ALLOWED", bind); } else if (PRTE_BIND_OVERLOAD_ALLOWED(binding)) { snprintf(ptr->buffers[ptr->cntr], PRTE_HWLOC_PRINT_MAX_SIZE, "%s:OVERLOAD-ALLOWED", bind); } else if (!PRTE_BINDING_REQUIRED(binding)) { snprintf(ptr->buffers[ptr->cntr], PRTE_HWLOC_PRINT_MAX_SIZE, "%s:IF-SUPPORTED", bind); } else { snprintf(ptr->buffers[ptr->cntr], PRTE_HWLOC_PRINT_MAX_SIZE, "%s", bind); } ret = ptr->buffers[ptr->cntr]; ptr->cntr++; return ret; } void prte_hwloc_build_map(hwloc_topology_t topo, hwloc_cpuset_t avail, bool use_hwthread_cpus, hwloc_bitmap_t coreset) { unsigned k, obj_index, core_index; hwloc_obj_t pu, core; /* the bits in the cpuset _always_ represent hwthreads, so * we have to manually determine which core each bit is under * so we can report the cpus in terms of "cores" */ /* start with the first set bit */ hwloc_bitmap_zero(coreset); k = hwloc_bitmap_first(avail); obj_index = 0; while (k != (unsigned) -1) { if (use_hwthread_cpus) { /* mark this thread as occupied */ hwloc_bitmap_set(coreset, k); } else { /* Go upward and find the core this PU belongs to */ pu = hwloc_get_obj_inside_cpuset_by_type(topo, avail, HWLOC_OBJ_PU, obj_index); core = pu; while (NULL != core && core->type != HWLOC_OBJ_CORE) { core = core->parent; } core_index = 0; if (NULL != core) { core_index = core->logical_index; } /* mark everything since the last place as * being empty */ hwloc_bitmap_set(coreset, core_index); } /* move to the next set bit */ k = hwloc_bitmap_next(avail, k); ++obj_index; } } /* formatting core/hwt binding information as xml elements */ static int bitmap_list_snprintf_exp(char *__hwloc_restrict buf, size_t buflen, const struct hwloc_bitmap_s *__hwloc_restrict set, char *type) { int ret = 0; char *tmp = buf; int prev = -1; ssize_t size = buflen; int res = -1; /* mark the end in case we do nothing later */ if (buflen > 0) { tmp[0] = '\0'; } while (1) { int begin, end; begin = hwloc_bitmap_next(set, prev); if (begin == -1) { break; } end = hwloc_bitmap_next_unset(set, begin); if (end == begin + 1) { res = snprintf(tmp, size, "%*c<%s>%d\n", 20, ' ', type, begin, type); } else if (end == -1) { res = snprintf(tmp, size, "%*c<%s>%d\n", 20, ' ', type, begin, type); } else { for (int i = begin; i <= end - 1; i++) { res = snprintf(tmp, size, "%*c<%s>%d\n", 20, ' ', type, i, type); if (i != (end - 1)) { tmp += res; } } } if (res < 0) { return -1; } ret += res; if (res >= size) { res = size > 0 ? (int)size - 1 : 0; } tmp += res; size -= res; if (end == -1) { break; } else { prev = end - 1; } } return ret; } /* * Output is undefined if a rank is bound to more than 1 package */ void prte_hwloc_get_binding_info(hwloc_const_cpuset_t cpuset, bool use_hwthread_cpus, hwloc_topology_t topo, int *pkgnum, char *cores, int sz) { int n, npkgs, npus, ncores; hwloc_cpuset_t avail, coreset = NULL; hwloc_obj_t pkg; bool bits_as_cores = false; /* if the cpuset is all zero, then something is wrong */ if (hwloc_bitmap_iszero(cpuset)) { snprintf(cores, sz, "\n%*c\n", 20, ' '); } /* if the cpuset includes all available cpus, and * the available cpus were not externally constrained, * then we are unbound */ avail = prte_hwloc_base_filter_cpus(topo); if (hwloc_bitmap_isequal(cpuset, avail) && hwloc_bitmap_isfull(avail)) { snprintf(cores, sz, "\n%*c\n", 20, ' '); } hwloc_bitmap_free(avail); /* get the number of packages in the topology */ npkgs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PACKAGE); avail = hwloc_bitmap_alloc(); npus = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PU); ncores = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE); if (npus == ncores && !use_hwthread_cpus) { /* the bits in this bitmap represent cores */ bits_as_cores = true; } if (!use_hwthread_cpus && !bits_as_cores) { coreset = hwloc_bitmap_alloc(); } /* binding happens within a package and not across packages */ for (n = 0; n < npkgs; n++) { pkg = prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_PACKAGE, n); /* see if we have any here */ hwloc_bitmap_and(avail, cpuset, pkg->cpuset); if (hwloc_bitmap_iszero(avail)) { continue; } if (bits_as_cores) { /* can just use the hwloc fn directly */ bitmap_list_snprintf_exp(cores, sz, avail, "core"); } else if (use_hwthread_cpus) { /* can just use the hwloc fn directly */ bitmap_list_snprintf_exp(cores, sz, avail, "hwt"); } else { prte_hwloc_build_map(topo, avail, use_hwthread_cpus | bits_as_cores, coreset); /* now print out the string */ bitmap_list_snprintf_exp(cores, sz, coreset, "core"); } *pkgnum = n; } hwloc_bitmap_free(avail); if (NULL != coreset) { hwloc_bitmap_free(coreset); } } static int compare_unsigned(const void *a, const void *b) { return (*(unsigned *)a - *(unsigned *)b); } /* generate a logical string output of a hwloc_cpuset_t */ static int build_map(char *answer, size_t size, hwloc_const_cpuset_t bitmap, bool physical, char *prefix, hwloc_topology_t topo) { unsigned indices[2048], id; int nsites = 0, n, start, end, idx; hwloc_obj_t pu; char tmp[128]; bool inrange, first, unique; unsigned val; for (id = hwloc_bitmap_first(bitmap); id != (unsigned)-1; id = hwloc_bitmap_next(bitmap, id)) { // the id's are for threads, but we want cores pu = hwloc_get_pu_obj_by_os_index(topo, id); // go upward to find the core that contains this pu while (NULL != pu && pu->type != HWLOC_OBJ_CORE) { pu = pu->parent; } if (NULL == pu) { pmix_show_help("help-prte-hwloc-base.txt", "pu-not-found", true, id); return PRTE_ERR_SILENT; } if (physical) { // record the physical site val = pu->os_index; } else { // record the logical site val = pu->logical_index; } // add it uniquely to the array of indices - it could be a duplicate unique = true; for (n=0; n < nsites; n++) { if (indices[n] == val) { unique = false; break; } } if (unique) { indices[nsites] = val; ++nsites; if (2048 == nsites) { pmix_show_help("help-prte-hwloc-base.txt", "too-many-sites", true); return PRTE_ERR_SILENT; } } } /* this should never happen as it would mean that the bitmap was * empty, which is something we checked before calling this function */ if (0 == nsites) { return PRTE_ERR_NOT_FOUND; } if (1 == nsites) { // only bound to one location - most common case snprintf(answer, size, "%s%u", prefix, indices[0]); return PRTE_SUCCESS; } // sort them qsort(indices, nsites, sizeof(unsigned), compare_unsigned); // parse through and look for ranges start = indices[0]; end = indices[0]; inrange = false; first = true; // prep the answer snprintf(answer, size, "%s", prefix); idx = strlen(prefix); for (n=1; n < nsites; n++) { // see if we are in a range if (1 == (indices[n]-end)) { inrange = true; end = indices[n]; continue; } // we are not in a range, or we are // at the end of a range if (inrange) { // we are at the end of the range if (start == end) { if (first) { snprintf(tmp, 128, "%u", start); first = false; } else { snprintf(tmp, 128, ",%u", start); } memcpy(&answer[idx], tmp, strlen(tmp)); idx += strlen(tmp); } else { if (first) { snprintf(tmp, 128, "%u-%u", start, end); first = false; } else { snprintf(tmp, 128, ",%u-%u", start, end); } memcpy(&answer[idx], tmp, strlen(tmp)); idx += strlen(tmp); } // mark the end of the range inrange = false; start = indices[n]; end = indices[n]; } else { if (first) { snprintf(tmp, 128, "%u", start); first = false; } else { snprintf(tmp, 128, ",%u", start); } memcpy(&answer[idx], tmp, strlen(tmp)); idx += strlen(tmp); inrange = false; start = indices[n]; end = indices[n]; } } // see if we have a dangling entry if (start == end) { if (first) { snprintf(tmp, 128, "%u", start); } else { snprintf(tmp, 128, ",%u", start); } memcpy(&answer[idx], tmp, strlen(tmp)); snprintf(tmp, 128, "%u", start); } else { if (first) { snprintf(tmp, 128, "%u-%u", start, end); first = false; } else { snprintf(tmp, 128, ",%u-%u", start, end); } memcpy(&answer[idx], tmp, strlen(tmp)); idx += strlen(tmp); } return PRTE_SUCCESS; } /* * Make a prettyprint string for a hwloc_cpuset_t */ char *prte_hwloc_base_cset2str(hwloc_const_cpuset_t cpuset, bool use_hwthread_cpus, bool physical, hwloc_topology_t topo) { int n, npkgs, npus, ncores; char tmp[2048], ans[4096]; hwloc_cpuset_t avail, coreset = NULL; char **output = NULL, *result; hwloc_obj_t pkg; bool bits_as_cores = false; int complete; char *prefix; /* if the cpuset is all zero, then something is wrong */ if (hwloc_bitmap_iszero(cpuset)) { return strdup("EMPTY CPUSET"); } /* if the cpuset includes all available cpus, and * the available cpus were not externally constrained, * then we are unbound */ avail = prte_hwloc_base_filter_cpus(topo); if (hwloc_bitmap_isequal(cpuset, avail) && hwloc_bitmap_isfull(avail)) { return strdup("UNBOUND"); } hwloc_bitmap_free(avail); /* get the number of packages in the topology */ npkgs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PACKAGE); avail = hwloc_bitmap_alloc(); npus = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PU); ncores = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_CORE); if (npus == ncores && !use_hwthread_cpus) { /* the bits in this bitmap represent cores */ bits_as_cores = true; } if (!use_hwthread_cpus && !bits_as_cores) { coreset = hwloc_bitmap_alloc(); } if (bits_as_cores || !use_hwthread_cpus) { if (physical) { prefix = "core:P"; } else { prefix = "core:L"; } } else { if (physical) { prefix = "hwt:P"; } else { prefix = "hwt:L"; } } for (n = 0; n < npkgs; n++) { memset(tmp, 0, sizeof(tmp)); memset(ans, 0, sizeof(ans)); pkg = prte_hwloc_base_get_obj_by_type(topo, HWLOC_OBJ_PACKAGE, n); /* see if we have any here */ hwloc_bitmap_and(avail, cpuset, pkg->cpuset); if (hwloc_bitmap_iszero(avail)) { continue; } if (bits_as_cores) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); snprintf(ans, 4096, "package[%d][%s%s]", n, prefix, tmp); } else if (use_hwthread_cpus) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); snprintf(ans, 4096, "package[%d][%s%s]", n, prefix, tmp); } else { // build the map for this cpuset complete = build_map(tmp, 2048, avail, physical, prefix, topo); if (PRTE_SUCCESS == complete) { snprintf(ans, 4096, "package[%d][%s]", n, tmp); } else { PMIx_Argv_free(output); return NULL; } } PMIx_Argv_append_nosize(&output, ans); } if (NULL != output) { result = PMIx_Argv_join(output, ' '); PMIx_Argv_free(output); } else { result = NULL; } hwloc_bitmap_free(avail); if (NULL != coreset) { hwloc_bitmap_free(coreset); } return result; } static char* construct_range(char **vals) { int n, cnt; char buf[4096], **ans = NULL, *str; if (NULL == vals) { str = strdup("-"); return str; } cnt = 1; for (n=0; NULL != vals[n]; n++) { if (NULL == vals[n+1]) { if (1 == cnt) { PMIx_Argv_append_nosize(&ans, vals[n]); } else { snprintf(buf, 4096, "%d:%s", cnt, vals[n]); PMIx_Argv_append_nosize(&ans, buf); } break; } if (0 == strcmp(vals[n], vals[n+1])) { cnt++; } else { if (1 == cnt) { PMIx_Argv_append_nosize(&ans, vals[n]); } else { snprintf(buf, 4096, "%d:%s", cnt, vals[n]); PMIx_Argv_append_nosize(&ans, buf); } cnt = 1; } } str = PMIx_Argv_join(ans, ','); return str; } char *prte_hwloc_base_get_topo_signature(hwloc_topology_t topo) { char *sig = NULL, *arch = NULL, *endian; hwloc_obj_t obj; unsigned i, nobjs, n, ncpus; char buffer[4096], **scratch = NULL, **answer = NULL; int rc; hwloc_cpuset_t avail, available; rc = hwloc_topology_export_synthetic(topo, buffer, 4096, HWLOC_TOPOLOGY_EXPORT_SYNTHETIC_FLAG_NO_ATTRS); if (0 > rc) { // create out own signature - start with packages scratch = NULL; available = hwloc_bitmap_alloc(); avail = prte_hwloc_base_filter_cpus(prte_hwloc_topology); nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_PACKAGE); for (n=0; n < nobjs; n++) { obj = prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_PACKAGE, n); hwloc_bitmap_and(available, avail, obj->cpuset); ncpus = hwloc_bitmap_weight(available); snprintf(buffer, 4096, "%u", ncpus); PMIx_Argv_append_nosize(&scratch, buffer); } sig = construct_range(scratch); snprintf(buffer, 4096, "PKG[%s]", sig); free(sig); PMIx_Argv_free(scratch); PMIx_Argv_append_nosize(&answer, buffer); // now account for NUMA scratch = NULL; nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_NUMANODE); for (n=0; n < nobjs; n++) { obj = prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_NUMANODE, n); hwloc_bitmap_and(available, avail, obj->cpuset); ncpus = hwloc_bitmap_weight(available); snprintf(buffer, 4096, "%u", ncpus); PMIx_Argv_append_nosize(&scratch, buffer); } sig = construct_range(scratch); snprintf(buffer, 4096, "NUMA[%s]", sig); free(sig); PMIx_Argv_free(scratch); PMIx_Argv_append_nosize(&answer, buffer); // L3caches scratch = NULL; nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L3CACHE); for (n=0; n < nobjs; n++) { obj = prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_L3CACHE, n); hwloc_bitmap_and(available, avail, obj->cpuset); ncpus = hwloc_bitmap_weight(available); snprintf(buffer, 4096, "%u", ncpus); PMIx_Argv_append_nosize(&scratch, buffer); } sig = construct_range(scratch); snprintf(buffer, 4096, "L3[%s]", sig); free(sig); PMIx_Argv_free(scratch); PMIx_Argv_append_nosize(&answer, buffer); // L2caches scratch = NULL; nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L2CACHE); for (n=0; n < nobjs; n++) { obj = prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_L2CACHE, n); hwloc_bitmap_and(available, avail, obj->cpuset); ncpus = hwloc_bitmap_weight(available); snprintf(buffer, 4096, "%u", ncpus); PMIx_Argv_append_nosize(&scratch, buffer); } sig = construct_range(scratch); snprintf(buffer, 4096, "L2[%s]", sig); free(sig); PMIx_Argv_free(scratch); PMIx_Argv_append_nosize(&answer, buffer); // L1caches scratch = NULL; nobjs = prte_hwloc_base_get_nbobjs_by_type(topo, HWLOC_OBJ_L1CACHE); for (n=0; n < nobjs; n++) { obj = prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_L1CACHE, n); hwloc_bitmap_and(available, avail, obj->cpuset); ncpus = hwloc_bitmap_weight(available); snprintf(buffer, 4096, "%u", ncpus); PMIx_Argv_append_nosize(&scratch, buffer); } sig = construct_range(scratch); snprintf(buffer, 4096, "L1[%s]", sig); free(sig); PMIx_Argv_free(scratch); PMIx_Argv_append_nosize(&answer, buffer); // setup the signature sig = PMIx_Argv_join(answer, ';'); snprintf(buffer, 4096, "%s", sig); free(sig); PMIx_Argv_free(answer); hwloc_bitmap_free(avail); } /* get the root object so we can add the processor architecture */ obj = hwloc_get_root_obj(topo); for (i = 0; i < obj->infos_count; i++) { if (0 == strcmp(obj->infos[i].name, "Architecture")) { arch = obj->infos[i].value; break; } } if (NULL == arch) { arch = "unknown"; } #ifdef __BYTE_ORDER # if __BYTE_ORDER == __LITTLE_ENDIAN endian = "le"; # else endian = "be"; # endif #else endian = "unknown"; #endif // form the final signature pmix_asprintf(&sig, "%s:%s:%s", buffer, arch, endian); return sig; } static int prte_hwloc_base_get_locality_string_by_depth(hwloc_topology_t topo, int d, hwloc_cpuset_t cpuset, hwloc_cpuset_t result) { hwloc_obj_t obj; unsigned width, w; /* get the width of the topology at this depth */ width = hwloc_get_nbobjs_by_depth(topo, d); if (0 == width) { return -1; } /* scan all objects at this depth to see if * the location overlaps with them */ for (w = 0; w < width; w++) { /* get the object at this depth/index */ obj = hwloc_get_obj_by_depth(topo, d, w); /* see if the location intersects with it */ if (hwloc_bitmap_intersects(obj->cpuset, cpuset)) { hwloc_bitmap_set(result, w); } } return 0; } char *prte_hwloc_base_get_locality_string(hwloc_topology_t topo, char *bitmap) { char *locality = NULL, *tmp, *t2; unsigned depth, d; hwloc_cpuset_t cpuset, result; hwloc_obj_type_t type; /* if this proc is not bound, then there is no locality. We * know it isn't bound if the cpuset is NULL, or if it is * all 1's */ if (NULL == bitmap) { return NULL; } cpuset = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(cpuset, bitmap); if (hwloc_bitmap_isfull(cpuset)) { hwloc_bitmap_free(cpuset); return NULL; } /* we are going to use a bitmap to save the results so * that we can use a hwloc utility to print them */ result = hwloc_bitmap_alloc(); /* get the max depth of the topology */ depth = hwloc_topology_get_depth(topo); /* start at the first depth below the top machine level */ for (d = 1; d < depth; d++) { /* get the object type at this depth */ type = hwloc_get_depth_type(topo, d); /* if it isn't one of interest, then ignore it */ if (HWLOC_OBJ_NUMANODE != type && HWLOC_OBJ_PACKAGE != type && HWLOC_OBJ_L1CACHE != type && HWLOC_OBJ_L2CACHE != type && HWLOC_OBJ_L3CACHE != type && HWLOC_OBJ_CORE != type && HWLOC_OBJ_PU != type) { continue; } if (prte_hwloc_base_get_locality_string_by_depth(topo, d, cpuset, result) < 0) { continue; } /* it should be impossible, but allow for the possibility * that we came up empty at this depth */ if (!hwloc_bitmap_iszero(result)) { hwloc_bitmap_list_asprintf(&tmp, result); switch (type) { case HWLOC_OBJ_NUMANODE: pmix_asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_PACKAGE: pmix_asprintf(&t2, "%sSK%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_L3CACHE: pmix_asprintf(&t2, "%sL3%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_L2CACHE: pmix_asprintf(&t2, "%sL2%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_L1CACHE: pmix_asprintf(&t2, "%sL1%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_CORE: pmix_asprintf(&t2, "%sCR%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; case HWLOC_OBJ_PU: pmix_asprintf(&t2, "%sHT%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; break; default: /* just ignore it */ break; } free(tmp); } hwloc_bitmap_zero(result); } if (prte_hwloc_base_get_locality_string_by_depth(topo, HWLOC_TYPE_DEPTH_NUMANODE, cpuset, result) == 0) { /* it should be impossible, but allow for the possibility * that we came up empty at this depth */ if (!hwloc_bitmap_iszero(result)) { hwloc_bitmap_list_asprintf(&tmp, result); pmix_asprintf(&t2, "%sNM%s:", (NULL == locality) ? "" : locality, tmp); if (NULL != locality) { free(locality); } locality = t2; free(tmp); } hwloc_bitmap_zero(result); } hwloc_bitmap_free(result); hwloc_bitmap_free(cpuset); /* remove the trailing colon */ if (NULL != locality) { locality[strlen(locality) - 1] = '\0'; } return locality; } char *prte_hwloc_base_get_location(char *locality, hwloc_obj_type_t type, unsigned index) { char **loc; char *srch, *ans = NULL; size_t n; PRTE_HIDE_UNUSED_PARAMS(index); if (NULL == locality) { return NULL; } switch (type) { case HWLOC_OBJ_NUMANODE: srch = "NM"; break; case HWLOC_OBJ_PACKAGE: srch = "SK"; break; case HWLOC_OBJ_L3CACHE: srch = "L3"; break; case HWLOC_OBJ_L2CACHE: srch = "L2"; break; case HWLOC_OBJ_L1CACHE: srch = "L1"; break; case HWLOC_OBJ_CORE: srch = "CR"; break; case HWLOC_OBJ_PU: srch = "HT"; break; default: return NULL; } loc = PMIx_Argv_split(locality, ':'); for (n = 0; NULL != loc[n]; n++) { if (0 == strncmp(loc[n], srch, 2)) { ans = strdup(&loc[n][2]); break; } } PMIx_Argv_free(loc); return ans; } prte_hwloc_locality_t prte_hwloc_compute_relative_locality(char *loc1, char *loc2) { prte_hwloc_locality_t locality; char **set1, **set2; hwloc_bitmap_t bit1, bit2; size_t n1, n2; /* start with what we know - they share a node on a cluster * NOTE: we may alter that latter part as hwloc's ability to * sense multi-cu, multi-cluster systems grows */ locality = PRTE_PROC_ON_NODE | PRTE_PROC_ON_HOST | PRTE_PROC_ON_CU | PRTE_PROC_ON_CLUSTER; /* if either location is NULL, then that isn't bound */ if (NULL == loc1 || NULL == loc2) { return locality; } set1 = PMIx_Argv_split(loc1, ':'); set2 = PMIx_Argv_split(loc2, ':'); bit1 = hwloc_bitmap_alloc(); bit2 = hwloc_bitmap_alloc(); /* check each matching type */ for (n1 = 0; NULL != set1[n1]; n1++) { /* convert the location into bitmap */ hwloc_bitmap_list_sscanf(bit1, &set1[n1][2]); /* find the matching type in set2 */ for (n2 = 0; NULL != set2[n2]; n2++) { if (0 == strncmp(set1[n1], set2[n2], 2)) { /* convert the location into bitmap */ hwloc_bitmap_list_sscanf(bit2, &set2[n2][2]); /* see if they intersect */ if (hwloc_bitmap_intersects(bit1, bit2)) { /* set the corresponding locality bit */ if (0 == strncmp(set1[n1], "SK", 2)) { locality |= PRTE_PROC_ON_PACKAGE; } else if (0 == strncmp(set1[n1], "NM", 2)) { locality |= PRTE_PROC_ON_NUMA; } else if (0 == strncmp(set1[n1], "L3", 2)) { locality |= PRTE_PROC_ON_L3CACHE; } else if (0 == strncmp(set1[n1], "L2", 2)) { locality |= PRTE_PROC_ON_L2CACHE; } else if (0 == strncmp(set1[n1], "L1", 2)) { locality |= PRTE_PROC_ON_L1CACHE; } else if (0 == strncmp(set1[n1], "CR", 2)) { locality |= PRTE_PROC_ON_CORE; } else if (0 == strncmp(set1[n1], "HT", 2)) { locality |= PRTE_PROC_ON_HWTHREAD; } else { /* should never happen */ pmix_output(0, "UNRECOGNIZED LOCALITY %s", set1[n1]); } } break; } } } PMIx_Argv_free(set1); PMIx_Argv_free(set2); hwloc_bitmap_free(bit1); hwloc_bitmap_free(bit2); return locality; } int prte_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen) { return hwloc_topology_export_xmlbuffer(topology, xmlpath, buflen, 0); } int prte_hwloc_base_topology_set_flags(hwloc_topology_t topology, unsigned long flags, bool io) { if (io) { int ret = hwloc_topology_set_io_types_filter(topology, HWLOC_TYPE_FILTER_KEEP_IMPORTANT); if (0 != ret) { return ret; } } // Blacklist the "gl" component due to potential conflicts. // See "https://github.com/open-mpi/ompi/issues/10025" for // an explanation hwloc_topology_set_components(topology, HWLOC_TOPOLOGY_COMPONENTS_FLAG_BLACKLIST, "gl"); return hwloc_topology_set_flags(topology, flags); } #define PRTE_HWLOC_MAX_STRING 2048 static void print_hwloc_obj(char **output, char *prefix, hwloc_topology_t topo, hwloc_obj_t obj) { hwloc_obj_t obj2; char string[1024], *tmp, *tmp2, *pfx; unsigned i; struct hwloc_topology_support *support; /* print the object type */ hwloc_obj_type_snprintf(string, 1024, obj, 1); pmix_asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix); pmix_asprintf(&tmp, "%sType: %s Number of child objects: %u%sName=%s", (NULL == prefix) ? "" : prefix, string, obj->arity, pfx, (NULL == obj->name) ? "NULL" : obj->name); if (0 < hwloc_obj_attr_snprintf(string, 1024, obj, pfx, 1)) { /* print the attributes */ pmix_asprintf(&tmp2, "%s%s%s", tmp, pfx, string); free(tmp); tmp = tmp2; } /* print the cpusets - apparently, some new HWLOC types don't * have cpusets, so protect ourselves here */ if (NULL != obj->cpuset) { hwloc_bitmap_snprintf(string, PRTE_HWLOC_MAX_STRING, obj->cpuset); pmix_asprintf(&tmp2, "%s%sCpuset: %s", tmp, pfx, string); free(tmp); tmp = tmp2; } if (HWLOC_OBJ_MACHINE == obj->type) { /* root level object - add support values */ support = (struct hwloc_topology_support *) hwloc_topology_get_support(topo); pmix_asprintf(&tmp2, "%s%sBind CPU proc: %s%sBind CPU thread: %s", tmp, pfx, (support->cpubind->set_thisproc_cpubind) ? "TRUE" : "FALSE", pfx, (support->cpubind->set_thisthread_cpubind) ? "TRUE" : "FALSE"); free(tmp); tmp = tmp2; pmix_asprintf(&tmp2, "%s%sBind MEM proc: %s%sBind MEM thread: %s", tmp, pfx, (support->membind->set_thisproc_membind) ? "TRUE" : "FALSE", pfx, (support->membind->set_thisthread_membind) ? "TRUE" : "FALSE"); free(tmp); tmp = tmp2; } pmix_asprintf(&tmp2, "%s%s\n", (NULL == *output) ? "" : *output, tmp); free(tmp); free(pfx); pmix_asprintf(&pfx, "%s\t", (NULL == prefix) ? "" : prefix); for (i = 0; i < obj->arity; i++) { obj2 = obj->children[i]; /* print the object */ print_hwloc_obj(&tmp2, pfx, topo, obj2); } free(pfx); if (NULL != *output) { free(*output); } *output = tmp2; } int prte_hwloc_print(char **output, char *prefix, hwloc_topology_t src) { hwloc_obj_t obj; char *tmp = NULL; /* get root object */ obj = hwloc_get_root_obj(src); /* print it */ print_hwloc_obj(&tmp, prefix, src, obj); *output = tmp; return PRTE_SUCCESS; } void prte_hwloc_base_reset_counters(void) { prte_topology_t *ptopo; hwloc_topology_t topo; hwloc_obj_type_t type; hwloc_obj_t obj; prte_hwloc_obj_data_t *objcnt; unsigned width, w; unsigned depth, d; int n; /* this can be a fairly expensive operation as we must traverse * all objects of interest in all topologies since we cannot * know which ones might have been used. Fortunately, we almost * always have only one topology, and there aren't that many * objects in it - so this normally goes fairly quickly */ for (n = 0; n < prte_node_topologies->size; n++) { ptopo = (prte_topology_t *) pmix_pointer_array_get_item(prte_node_topologies, n); if (NULL == ptopo) { continue; } topo = ptopo->topo; /* get the max depth of the topology */ depth = hwloc_topology_get_depth(topo); /* start at the first depth below the top machine level */ for (d = 1; d < depth; d++) { /* get the object type at this depth */ type = hwloc_get_depth_type(topo, d); /* if it isn't one of interest, then ignore it */ if (HWLOC_OBJ_NUMANODE != type && HWLOC_OBJ_PACKAGE != type && HWLOC_OBJ_L1CACHE != type && HWLOC_OBJ_L2CACHE != type && HWLOC_OBJ_L3CACHE != type && HWLOC_OBJ_CORE != type && HWLOC_OBJ_PU != type) { continue; } /* get the width of the topology at this depth */ width = hwloc_get_nbobjs_by_depth(topo, d); if (0 == width) { continue; } /* scan all objects at this depth to see if * the location overlaps with them */ for (w = 0; w < width; w++) { /* get the object at this depth/index */ obj = hwloc_get_obj_by_depth(topo, d, w); if (NULL != obj->userdata) { objcnt = (prte_hwloc_obj_data_t*)obj->userdata; objcnt->nprocs = 0; } } } } } prrte-4.1.1~rc2/src/common_sym_whitelist.txt0000664000175000017500000000040715203542132021444 0ustar alastairalastair# Ignore symbols in libopen-rte that are auto-generated and we can't # do anything about them (e.g., flex/bison symbols). prte_util_hostfile_leng prte_util_hostfile_text prte_util_keyval_yyleng prte_util_keyval_yytext pmix_show_help_yyleng pmix_show_help_yytext prrte-4.1.1~rc2/src/event/0000775000175000017500000000000015203542132015547 5ustar alastairalastairprrte-4.1.1~rc2/src/event/Makefile.am0000664000175000017500000000213015203542132017577 0ustar alastairalastair# -*- makefile -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. # Copyright (c) 2014-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2018-2020 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This makefile.am does not stand on its own - it is included from src/Makefile.am headers += \ event/event-internal.h libprrte_la_SOURCES += \ event/event.c prrte-4.1.1~rc2/src/event/event.c0000664000175000017500000000345515203542132017043 0ustar alastairalastair/* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/include/constants.h" #include "src/event/event-internal.h" /* * Globals */ prte_event_base_t *prte_sync_event_base = NULL; static bool initialized = false; int prte_event_base_open(void) { if (initialized) { return PRTE_SUCCESS; } /* Declare our intent to use threads */ prte_event_use_threads(); /* get our event base */ if (NULL == (prte_sync_event_base = event_base_new())) { return PRTE_ERROR; } /* PRTE tools "block" in their own loop over the event * base, so no progress thread is required */ prte_event_base = prte_sync_event_base; initialized = true; return PRTE_SUCCESS; } int prte_event_base_close(void) { if (!initialized) { return PRTE_SUCCESS; } prte_event_base_free(prte_sync_event_base); initialized = false; return PRTE_SUCCESS; } prte_event_t *prte_event_alloc(void) { prte_event_t *ev; ev = (prte_event_t *) malloc(sizeof(prte_event_t)); return ev; } int prte_event_assign(struct event *ev, prte_event_base_t *evbase, int fd, short arg, event_callback_fn cbfn, void *cbd) { #if PRTE_HAVE_LIBEV event_set(ev, fd, arg, cbfn, cbd); event_base_set(evbase, ev); #else event_assign(ev, evbase, fd, arg, cbfn, cbd); #endif return 0; } PMIX_CLASS_INSTANCE(prte_event_list_item_t, pmix_list_item_t, NULL, NULL); prrte-4.1.1~rc2/src/event/event-internal.h0000664000175000017500000001234615203542132020661 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow */ #ifndef PRTE_MCA_EVENT_H #define PRTE_MCA_EVENT_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #include #include #include #if !PRTE_HAVE_LIBEV # include #endif #include "src/class/pmix_list.h" #include "src/util/pmix_output.h" typedef event_callback_fn prte_event_cbfunc_t; BEGIN_C_DECLS #define PRTE_EVENT_SIGNAL(ev) prte_event_get_signal(ev) #define PRTE_TIMEOUT_DEFAULT \ { \ 1, 0 \ } typedef struct event_base prte_event_base_t; typedef struct event prte_event_t; PRTE_EXPORT extern prte_event_base_t *prte_sync_event_base; PRTE_EXPORT extern prte_event_base_t *prte_event_base; PRTE_EXPORT int prte_event_base_open(void); PRTE_EXPORT int prte_event_base_close(void); PRTE_EXPORT prte_event_t *prte_event_alloc(void); #define PRTE_EV_TIMEOUT EV_TIMEOUT #define PRTE_EV_READ EV_READ #define PRTE_EV_WRITE EV_WRITE #define PRTE_EV_SIGNAL EV_SIGNAL /* Persistent event: won't get removed automatically when activated. */ #define PRTE_EV_PERSIST EV_PERSIST #define PRTE_EVLOOP_ONCE EVLOOP_ONCE /**< Block at most once. */ #define PRTE_EVLOOP_NONBLOCK EVLOOP_NONBLOCK /**< Do not block. */ #define prte_event_base_create() event_base_new() #define prte_event_base_free(x) event_base_free(x) #define prte_event_reinit(b) event_reinit((b)) #define prte_event_base_init_common_timeout (b, t) event_base_init_common_timeout((b), (t)) #if PRTE_HAVE_LIBEV # define prte_event_use_threads() # define prte_event_free(b) free(b) # define prte_event_get_signal(x) (x)->ev_fd #else /* thread support APIs */ # define prte_event_use_threads() evthread_use_pthreads() # define prte_event_free(x) event_free(x) # define prte_event_get_signal(x) event_get_signal(x) #endif /* Event priority APIs */ #define prte_event_base_priority_init(b, n) event_base_priority_init((b), (n)) /* Basic event APIs */ #define prte_event_enable_debug_mode() event_enable_debug_mode() PRTE_EXPORT int prte_event_assign(struct event *ev, prte_event_base_t *evbase, int fd, short arg, event_callback_fn cbfn, void *cbd); #define prte_event_set(b, x, fd, fg, cb, arg) \ prte_event_assign((x), (b), (fd), (fg), (event_callback_fn)(cb), (arg)) #if PRTE_HAVE_LIBEV PRTE_EXPORT int prte_event_add(struct event *ev, struct timeval *tv); PRTE_EXPORT int prte_event_del(struct event *ev); PRTE_EXPORT void prte_event_active(struct event *ev, int res, short ncalls); PRTE_EXPORT void prte_event_base_loopexit(prte_event_base_t *b); #else # define prte_event_add(ev, tv) event_add((ev), (tv)) # define prte_event_del(ev) event_del((ev)) # define prte_event_active(x, y, z) event_active((x), (y), (z)) # define prte_event_base_loopexit(b) event_base_loopexit(b, NULL) #endif PRTE_EXPORT prte_event_t *prte_event_new(prte_event_base_t *b, int fd, short fg, event_callback_fn cbfn, void *cbd); /* Timer APIs */ #define prte_event_evtimer_new(b, cb, arg) prte_event_new((b), -1, 0, (cb), (arg)) #define prte_event_evtimer_add(x, tv) prte_event_add((x), (tv)) #define prte_event_evtimer_set(b, x, cb, arg) \ prte_event_assign((x), (b), -1, 0, (event_callback_fn)(cb), (arg)) #define prte_event_evtimer_del(x) prte_event_del((x)) #define prte_event_evtimer_pending(x, tv) event_pending((x), EV_TIMEOUT, (tv)) #define prte_event_evtimer_initialized(x) event_initialized((x)) /* Signal APIs */ #define prte_event_signal_add(x, tv) event_add((x), (tv)) #define prte_event_signal_set(b, x, fd, cb, arg) \ prte_event_assign((x), (b), (fd), EV_SIGNAL | EV_PERSIST, (event_callback_fn)(cb), (arg)) #define prte_event_signal_del(x) event_del((x)) #define prte_event_signal_pending(x, tv) event_pending((x), EV_SIGNAL, (tv)) #define prte_event_signal_initalized(x) event_initialized((x)) #define prte_event_loop(b, fg) event_base_loop((b), (fg)) typedef struct { pmix_list_item_t super; prte_event_t ev; } prte_event_list_item_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_event_list_item_t); /* define a threadshift macro */ #define PRTE_PMIX_THREADSHIFT(x, eb, f) \ do { \ prte_event_set((eb), &((x)->ev), -1, PRTE_EV_WRITE, (f), (x)); \ PMIX_POST_OBJECT((x)); \ prte_event_active(&((x)->ev), PRTE_EV_WRITE, 1); \ } while (0) END_C_DECLS #endif /* PRTE_EVENT_H_ */ prrte-4.1.1~rc2/src/pmix/0000775000175000017500000000000015203542132015403 5ustar alastairalastairprrte-4.1.1~rc2/src/pmix/pmix.c0000664000175000017500000003412415203542132016530 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "src/include/constants.h" #include #include #include #ifdef HAVE_UNISTD_H # include #endif #include "src/class/pmix_hash_table.h" #include "src/mca/plm/base/plm_private.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/proc_info.h" pmix_status_t prte_pmix_convert_rc(int rc) { switch (rc) { case PRTE_ERR_HEARTBEAT_ALERT: case PRTE_ERR_FILE_ALERT: case PRTE_ERR_HEARTBEAT_LOST: case PRTE_ERR_SENSOR_LIMIT_EXCEEDED: return PMIX_ERR_JOB_SENSOR_BOUND_EXCEEDED; case PRTE_ERR_NO_EXE_SPECIFIED: case PRTE_ERR_NO_APP_SPECIFIED: return PMIX_ERR_JOB_NO_EXE_SPECIFIED; case PRTE_ERR_SLOT_LIST_RANGE: case PRTE_ERR_TOPO_SOCKET_NOT_SUPPORTED: case PRTE_ERR_INVALID_PHYS_CPU: case PRTE_ERR_TOPO_CORE_NOT_SUPPORTED: case PRTE_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED: case PRTE_ERR_MULTIPLE_AFFINITIES: case PRTE_ERR_FAILED_TO_MAP: return PMIX_ERR_JOB_FAILED_TO_MAP; case PRTE_ERR_JOB_CANCELLED: return PMIX_ERR_JOB_CANCELED; case PRTE_ERR_DEBUGGER_RELEASE: return PMIX_ERR_DEBUGGER_RELEASE; case PRTE_ERR_HANDLERS_COMPLETE: return PMIX_EVENT_ACTION_COMPLETE; case PRTE_ERR_PROC_ABORTED: return PMIX_ERR_PROC_ABORTED; case PRTE_ERR_PROC_REQUESTED_ABORT: return PMIX_ERR_PROC_REQUESTED_ABORT; case PRTE_ERR_PROC_ABORTING: return PMIX_ERR_PROC_ABORTING; case PRTE_ERR_NODE_DOWN: return PMIX_ERR_NODE_DOWN; case PRTE_ERR_NODE_OFFLINE: return PMIX_ERR_NODE_OFFLINE; case PRTE_ERR_JOB_TERMINATED: return PMIX_EVENT_JOB_END; case PRTE_ERR_PROC_RESTART: return PMIX_ERR_PROC_RESTART; case PRTE_ERR_PROC_CHECKPOINT: return PMIX_ERR_PROC_CHECKPOINT; case PRTE_ERR_PROC_MIGRATE: return PMIX_ERR_PROC_MIGRATE; case PRTE_ERR_EVENT_REGISTRATION: return PMIX_ERR_EVENT_REGISTRATION; case PRTE_ERR_NOT_IMPLEMENTED: case PRTE_ERR_NOT_SUPPORTED: return PMIX_ERR_NOT_SUPPORTED; case PRTE_ERR_NOT_FOUND: return PMIX_ERR_NOT_FOUND; case PRTE_ERR_PERM: case PRTE_ERR_UNREACH: case PRTE_ERR_SERVER_NOT_AVAIL: return PMIX_ERR_UNREACH; case PRTE_ERR_BAD_PARAM: return PMIX_ERR_BAD_PARAM; case PRTE_ERR_SYS_LIMITS_PIPES: case PRTE_ERR_SYS_LIMITS_CHILDREN: case PRTE_ERR_SOCKET_NOT_AVAILABLE: case PRTE_ERR_NOT_ENOUGH_CORES: case PRTE_ERR_NOT_ENOUGH_SOCKETS: return PMIX_ERR_JOB_INSUFFICIENT_RESOURCES; case PRTE_ERR_PIPE_READ_FAILURE: return PMIX_ERR_JOB_SYS_OP_FAILED; case PRTE_ERR_OUT_OF_RESOURCE: return PMIX_ERR_OUT_OF_RESOURCE; case PRTE_ERR_DATA_VALUE_NOT_FOUND: return PMIX_ERR_DATA_VALUE_NOT_FOUND; case PRTE_ERR_WDIR_NOT_FOUND: return PMIX_ERR_JOB_WDIR_NOT_FOUND; case PRTE_ERR_EXE_NOT_FOUND: case PRTE_ERR_EXE_NOT_ACCESSIBLE: return PMIX_ERR_JOB_EXE_NOT_FOUND; case PRTE_ERR_TIMEOUT: return PMIX_ERR_TIMEOUT; case PRTE_ERR_WOULD_BLOCK: return PMIX_ERR_WOULD_BLOCK; case PRTE_EXISTS: return PMIX_EXISTS; case PRTE_ERR_PARTIAL_SUCCESS: return PMIX_QUERY_PARTIAL_SUCCESS; case PRTE_ERR_MODEL_DECLARED: return PMIX_MODEL_DECLARED; case PRTE_ERROR: return PMIX_ERROR; case PRTE_SUCCESS: return PMIX_SUCCESS; default: return PMIX_ERROR; } } int prte_pmix_convert_status(pmix_status_t status) { switch (status) { case PMIX_ERR_DEBUGGER_RELEASE: return PRTE_ERR_DEBUGGER_RELEASE; case PMIX_EVENT_ACTION_COMPLETE: return PRTE_ERR_HANDLERS_COMPLETE; case PMIX_ERR_PROC_ABORTED: return PRTE_ERR_PROC_ABORTED; case PMIX_ERR_PROC_REQUESTED_ABORT: return PRTE_ERR_PROC_REQUESTED_ABORT; case PMIX_ERR_PROC_ABORTING: return PRTE_ERR_PROC_ABORTING; case PMIX_ERR_NODE_DOWN: return PRTE_ERR_NODE_DOWN; case PMIX_ERR_NODE_OFFLINE: return PRTE_ERR_NODE_OFFLINE; case PMIX_EVENT_JOB_END: return PRTE_ERR_JOB_TERMINATED; case PMIX_ERR_PROC_RESTART: return PRTE_ERR_PROC_RESTART; case PMIX_ERR_PROC_CHECKPOINT: return PRTE_ERR_PROC_CHECKPOINT; case PMIX_ERR_PROC_MIGRATE: return PRTE_ERR_PROC_MIGRATE; case PMIX_ERR_EVENT_REGISTRATION: return PRTE_ERR_EVENT_REGISTRATION; case PMIX_ERR_NOT_SUPPORTED: return PRTE_ERR_NOT_SUPPORTED; case PMIX_ERR_NOT_FOUND: return PRTE_ERR_NOT_FOUND; case PMIX_ERR_OUT_OF_RESOURCE: return PRTE_ERR_OUT_OF_RESOURCE; case PMIX_ERR_INIT: return PRTE_ERROR; case PMIX_ERR_BAD_PARAM: return PRTE_ERR_BAD_PARAM; case PMIX_ERR_UNREACH: case PMIX_ERR_NO_PERMISSIONS: return PRTE_ERR_UNREACH; case PMIX_ERR_TIMEOUT: return PRTE_ERR_TIMEOUT; case PMIX_ERR_WOULD_BLOCK: return PRTE_ERR_WOULD_BLOCK; case PMIX_ERR_LOST_CONNECTION: return PRTE_ERR_COMM_FAILURE; case PMIX_EXISTS: return PRTE_EXISTS; case PMIX_QUERY_PARTIAL_SUCCESS: return PRTE_ERR_PARTIAL_SUCCESS; case PMIX_MONITOR_HEARTBEAT_ALERT: return PRTE_ERR_HEARTBEAT_ALERT; case PMIX_MONITOR_FILE_ALERT: return PRTE_ERR_FILE_ALERT; case PMIX_MODEL_DECLARED: return PRTE_ERR_MODEL_DECLARED; case PMIX_ERROR: return PRTE_ERROR; case PMIX_ERR_SILENT: return PRTE_ERR_SILENT; case PMIX_SUCCESS: case PMIX_OPERATION_SUCCEEDED: return PRTE_SUCCESS; case PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER: return PRTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER; default: return status; } } pmix_proc_state_t prte_pmix_convert_state(int state) { switch (state) { case 0: return PMIX_PROC_STATE_UNDEF; case 1: return PMIX_PROC_STATE_LAUNCH_UNDERWAY; case 2: return PMIX_PROC_STATE_RESTART; case 3: return PMIX_PROC_STATE_TERMINATE; case 4: return PMIX_PROC_STATE_RUNNING; case 5: return PMIX_PROC_STATE_CONNECTED; case 51: return PMIX_PROC_STATE_KILLED_BY_CMD; case 52: return PMIX_PROC_STATE_ABORTED; case 53: return PMIX_PROC_STATE_FAILED_TO_START; case 54: return PMIX_PROC_STATE_ABORTED_BY_SIG; case 55: return PMIX_PROC_STATE_TERM_WO_SYNC; case 56: return PMIX_PROC_STATE_COMM_FAILED; case 58: return PMIX_PROC_STATE_CALLED_ABORT; case 59: return PMIX_PROC_STATE_MIGRATING; case 61: return PMIX_PROC_STATE_CANNOT_RESTART; case 62: return PMIX_PROC_STATE_TERM_NON_ZERO; case 63: return PMIX_PROC_STATE_FAILED_TO_LAUNCH; default: return PMIX_PROC_STATE_UNDEF; } } int prte_pmix_convert_pstate(pmix_proc_state_t state) { switch (state) { case PMIX_PROC_STATE_UNDEF: return 0; case PMIX_PROC_STATE_PREPPED: case PMIX_PROC_STATE_LAUNCH_UNDERWAY: return 1; case PMIX_PROC_STATE_RESTART: return 2; case PMIX_PROC_STATE_TERMINATE: return 3; case PMIX_PROC_STATE_RUNNING: return 4; case PMIX_PROC_STATE_CONNECTED: return 5; case PMIX_PROC_STATE_UNTERMINATED: return 15; case PMIX_PROC_STATE_TERMINATED: return 20; case PMIX_PROC_STATE_KILLED_BY_CMD: return 51; case PMIX_PROC_STATE_ABORTED: return 52; case PMIX_PROC_STATE_FAILED_TO_START: return 53; case PMIX_PROC_STATE_ABORTED_BY_SIG: return 54; case PMIX_PROC_STATE_TERM_WO_SYNC: return 55; case PMIX_PROC_STATE_COMM_FAILED: return 56; case PMIX_PROC_STATE_CALLED_ABORT: return 58; case PMIX_PROC_STATE_MIGRATING: return 60; case PMIX_PROC_STATE_CANNOT_RESTART: return 61; case PMIX_PROC_STATE_TERM_NON_ZERO: return 62; case PMIX_PROC_STATE_FAILED_TO_LAUNCH: return 63; default: return 0; // undef } } pmix_status_t prte_pmix_convert_job_state_to_error(int state) { switch (state) { case PRTE_JOB_STATE_ALLOC_FAILED: return PMIX_ERR_JOB_ALLOC_FAILED; case PRTE_JOB_STATE_MAP_FAILED: return PMIX_ERR_JOB_FAILED_TO_MAP; case PRTE_JOB_STATE_NEVER_LAUNCHED: case PRTE_JOB_STATE_FAILED_TO_LAUNCH: case PRTE_JOB_STATE_FAILED_TO_START: case PRTE_JOB_STATE_CANNOT_LAUNCH: return PMIX_ERR_JOB_FAILED_TO_LAUNCH; case PRTE_JOB_STATE_KILLED_BY_CMD: return PMIX_ERR_JOB_CANCELED; case PRTE_JOB_STATE_ABORTED: case PRTE_JOB_STATE_CALLED_ABORT: case PRTE_JOB_STATE_SILENT_ABORT: return PMIX_ERR_JOB_ABORTED; case PRTE_JOB_STATE_ABORTED_BY_SIG: return PMIX_ERR_JOB_ABORTED_BY_SIG; case PRTE_JOB_STATE_ABORTED_WO_SYNC: return PMIX_ERR_JOB_TERM_WO_SYNC; case PRTE_JOB_STATE_TERMINATED: return PMIX_EVENT_JOB_END; default: return PMIX_ERROR; } } pmix_status_t prte_pmix_convert_proc_state_to_error(int state) { switch (state) { case PRTE_PROC_STATE_KILLED_BY_CMD: return PMIX_ERR_JOB_CANCELED; case PRTE_PROC_STATE_ABORTED: case PRTE_PROC_STATE_CALLED_ABORT: return PMIX_ERR_JOB_ABORTED; case PRTE_PROC_STATE_ABORTED_BY_SIG: return PMIX_ERR_JOB_ABORTED_BY_SIG; case PRTE_PROC_STATE_FAILED_TO_LAUNCH: case PRTE_PROC_STATE_FAILED_TO_START: return PMIX_ERR_JOB_FAILED_TO_LAUNCH; case PRTE_PROC_STATE_TERM_WO_SYNC: return PMIX_ERR_JOB_TERM_WO_SYNC; case PRTE_PROC_STATE_COMM_FAILED: case PRTE_PROC_STATE_UNABLE_TO_SEND_MSG: case PRTE_PROC_STATE_LIFELINE_LOST: case PRTE_PROC_STATE_NO_PATH_TO_TARGET: case PRTE_PROC_STATE_FAILED_TO_CONNECT: case PRTE_PROC_STATE_PEER_UNKNOWN: return PMIX_ERR_COMM_FAILURE; case PRTE_PROC_STATE_CANNOT_RESTART: return PMIX_ERR_PROC_RESTART; case PRTE_PROC_STATE_TERM_NON_ZERO: return PMIX_ERR_JOB_NON_ZERO_TERM; case PRTE_PROC_STATE_SENSOR_BOUND_EXCEEDED: return PMIX_ERR_JOB_SENSOR_BOUND_EXCEEDED; default: return PMIX_ERROR; } } static void cleanup_cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, void *cbdata, pmix_release_cbfunc_t release_fn, void *release_cbdata) { prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(info, ninfo); PMIX_POST_OBJECT(lk); /* let the library release the data and cleanup from * the operation */ if (NULL != release_fn) { release_fn(release_cbdata); } /* release the block */ lk->status = status; PRTE_PMIX_WAKEUP_THREAD(lk); } int prte_pmix_register_cleanup(char *path, bool directory, bool ignore, bool jobscope) { prte_pmix_lock_t lk; pmix_info_t pinfo[3]; size_t n, ninfo = 0; pmix_status_t rc, ret; PRTE_PMIX_CONSTRUCT_LOCK(&lk); if (ignore) { /* they want this path ignored */ PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_CLEANUP_IGNORE, path, PMIX_STRING); ++ninfo; } else { if (directory) { PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_REGISTER_CLEANUP_DIR, path, PMIX_STRING); ++ninfo; /* recursively cleanup directories */ PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_CLEANUP_RECURSIVE, NULL, PMIX_BOOL); ++ninfo; } else { /* order cleanup of the provided path */ PMIX_INFO_LOAD(&pinfo[ninfo], PMIX_REGISTER_CLEANUP, path, PMIX_STRING); ++ninfo; } } /* if they want this applied to the job, then indicate so */ if (jobscope) { rc = PMIx_Job_control_nb(NULL, 0, pinfo, ninfo, cleanup_cbfunc, (void *) &lk); } else { rc = PMIx_Job_control_nb(PRTE_PROC_MY_NAME, 1, pinfo, ninfo, cleanup_cbfunc, (void *) &lk); } if (PMIX_SUCCESS != rc) { ret = rc; } else { PRTE_PMIX_WAIT_THREAD(&lk); ret = lk.status; } PRTE_PMIX_DESTRUCT_LOCK(&lk); for (n = 0; n < ninfo; n++) { PMIX_INFO_DESTRUCT(&pinfo[n]); } return ret; } /* CLASS INSTANTIATIONS */ static void acon(prte_pmix_app_t *p) { PMIX_APP_CONSTRUCT(&p->app); PMIX_INFO_LIST_START(p->info); } static void ades(prte_pmix_app_t *p) { PMIX_APP_DESTRUCT(&p->app); PMIX_INFO_LIST_RELEASE(p->info); } PMIX_CLASS_INSTANCE(prte_pmix_app_t, pmix_list_item_t, acon, ades); static void infoitmcon(prte_info_item_t *p) { PMIX_INFO_CONSTRUCT(&p->info); } static void infoitdecon(prte_info_item_t *p) { PMIX_INFO_DESTRUCT(&p->info); } PRTE_EXPORT PMIX_CLASS_INSTANCE(prte_info_item_t, pmix_list_item_t, infoitmcon, infoitdecon); static void arritmcon(prte_info_array_item_t *p) { PMIX_CONSTRUCT(&p->infolist, pmix_list_t); } static void arritdecon(prte_info_array_item_t *p) { PMIX_LIST_DESTRUCT(&p->infolist); } PRTE_EXPORT PMIX_CLASS_INSTANCE(prte_info_array_item_t, pmix_list_item_t, arritmcon, arritdecon); static void pvcon(prte_value_t *p) { PMIX_VALUE_CONSTRUCT(&p->value); } static void pvdes(prte_value_t *p) { PMIX_VALUE_DESTRUCT(&p->value); } PRTE_EXPORT PMIX_CLASS_INSTANCE(prte_value_t, pmix_list_item_t, pvcon, pvdes); prrte-4.1.1~rc2/src/pmix/pmix-internal.h0000664000175000017500000002614115203542132020347 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PMIX_H #define PRTE_PMIX_H #include "prte_config.h" #ifdef HAVE_SYS_UN_H # include #endif #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/include/hash_string.h" #include "src/mca/mca.h" #include "src/threads/pmix_threads.h" #include "src/util/error.h" #include "src/include/pmix_globals.h" #include "src/util/pmix_error.h" #include "src/util/pmix_name_fns.h" #include "src/util/pmix_printf.h" #include "src/util/proc_info.h" #include #include #include #include BEGIN_C_DECLS PRTE_EXPORT extern int prte_pmix_verbose_output; typedef struct { pmix_list_item_t super; pmix_app_t app; void *info; } prte_pmix_app_t; PMIX_CLASS_DECLARATION(prte_pmix_app_t); /* define another caddy for putting statically defined * pmix_info_t objects on a list */ typedef struct { pmix_list_item_t super; pmix_info_t info; } prte_info_item_t; PMIX_CLASS_DECLARATION(prte_info_item_t); typedef struct { pmix_list_item_t super; pmix_list_t infolist; } prte_info_array_item_t; PMIX_CLASS_DECLARATION(prte_info_array_item_t); typedef struct { pmix_mutex_t mutex; pthread_cond_t cond; volatile bool active; int status; char *msg; } prte_pmix_lock_t; typedef struct { pmix_list_item_t super; pmix_value_t value; } prte_value_t; PMIX_CLASS_DECLARATION(prte_value_t); #if !defined(WORDS_BIGENDIAN) # define PMIX_PROC_NTOH(guid) pmix_proc_ntoh_intr(&(guid)) static inline __prte_attribute_always_inline__ void pmix_proc_ntoh_intr(pmix_proc_t *name) { name->rank = ntohl(name->rank); } # define PMIX_PROC_HTON(guid) pmix_proc_hton_intr(&(guid)) static inline __prte_attribute_always_inline__ void pmix_proc_hton_intr(pmix_proc_t *name) { name->rank = htonl(name->rank); } #else # define PMIX_PROC_NTOH(guid) # define PMIX_PROC_HTON(guid) #endif #define prte_pmix_condition_wait(a, b) pthread_cond_wait(a, &(b)->m_lock_pthread) #define PRTE_PMIX_CONSTRUCT_LOCK(l) \ do { \ PMIX_CONSTRUCT(&(l)->mutex, pmix_mutex_t); \ pmix_mutex_lock(&(l)->mutex); \ pthread_cond_init(&(l)->cond, NULL); \ (l)->active = true; \ (l)->status = 0; \ (l)->msg = NULL; \ PMIX_POST_OBJECT((l)); \ pmix_mutex_unlock(&(l)->mutex); \ } while (0) #define PRTE_PMIX_DESTRUCT_LOCK(l) \ do { \ PMIX_ACQUIRE_OBJECT((l)); \ PMIX_DESTRUCT(&(l)->mutex); \ pthread_cond_destroy(&(l)->cond); \ if (NULL != (l)->msg) { \ free((l)->msg); \ } \ } while (0) #if PRTE_ENABLE_DEBUG # define PRTE_PMIX_ACQUIRE_THREAD(lck) \ do { \ pmix_mutex_lock(&(lck)->mutex); \ while ((lck)->active) { \ prte_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \ } \ (lck)->active = true; \ } while (0) #else # define PRTE_PMIX_ACQUIRE_THREAD(lck) \ do { \ pmix_mutex_lock(&(lck)->mutex); \ while ((lck)->active) { \ prte_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \ } \ (lck)->active = true; \ } while (0) #endif #if PRTE_ENABLE_DEBUG # define PRTE_PMIX_WAIT_THREAD(lck) \ do { \ pmix_mutex_lock(&(lck)->mutex); \ while ((lck)->active) { \ prte_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \ } \ PMIX_ACQUIRE_OBJECT(&lck); \ pmix_mutex_unlock(&(lck)->mutex); \ } while (0) #else # define PRTE_PMIX_WAIT_THREAD(lck) \ do { \ pmix_mutex_lock(&(lck)->mutex); \ while ((lck)->active) { \ prte_pmix_condition_wait(&(lck)->cond, &(lck)->mutex); \ } \ PMIX_ACQUIRE_OBJECT(lck); \ pmix_mutex_unlock(&(lck)->mutex); \ } while (0) #endif #if PRTE_ENABLE_DEBUG # define PRTE_PMIX_RELEASE_THREAD(lck) \ do { \ (lck)->active = false; \ pthread_cond_signal(&(lck)->cond); \ pmix_mutex_unlock(&(lck)->mutex); \ } while (0) #else # define PRTE_PMIX_RELEASE_THREAD(lck) \ do { \ assert(0 != pmix_mutex_trylock(&(lck)->mutex)); \ (lck)->active = false; \ pthread_cond_signal(&(lck)->cond); \ pmix_mutex_unlock(&(lck)->mutex); \ } while (0) #endif #define PRTE_PMIX_WAKEUP_THREAD(lck) \ do { \ pmix_mutex_lock(&(lck)->mutex); \ (lck)->active = false; \ PMIX_POST_OBJECT(lck); \ pthread_cond_signal(&(lck)->cond); \ pmix_mutex_unlock(&(lck)->mutex); \ } while (0) /* * Count the hash for the the external RM */ #define PRTE_HASH_JOBID(str, hash) \ { \ PRTE_HASH_STR(str, hash); \ hash &= ~(0x8000); \ } /** * Provide a simplified macro for retrieving modex data * from another process when we don't want the PMIx module * to request it from the server if not found: * * r - the integer return status from the modex op (int) * s - string key (char*) * p - pointer to the pmix_proc_t of the proc that posted * the data (pmix_proc_t*) * d - pointer to a location wherein the data object * is to be returned * t - the expected data type */ #define PRTE_MODEX_RECV_VALUE_OPTIONAL(r, s, p, d, t) \ do { \ pmix_value_t *_kv = NULL; \ pmix_info_t _info; \ size_t _sz; \ PMIX_OUTPUT_VERBOSE((1, prte_pmix_verbose_output, \ "%s[%s:%d] MODEX RECV VALUE OPTIONAL FOR PROC %s KEY %s", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__, __LINE__, \ PRTE_NAME_PRINT((p)), (s))); \ PMIX_INFO_LOAD(&_info, PMIX_OPTIONAL, NULL, PMIX_BOOL); \ (r) = PMIx_Get((p), (s), &(_info), 1, &(_kv)); \ if (NULL == _kv) { \ (r) = PMIX_ERR_NOT_FOUND; \ } else if (_kv->type != (t)) { \ (r) = PMIX_ERR_TYPE_MISMATCH; \ } else if (PMIX_SUCCESS == (r)) { \ PMIX_VALUE_UNLOAD((r), _kv, (void **) (d), &_sz); \ } \ if (NULL != _kv) { \ PMIX_VALUE_RELEASE(_kv); \ } \ } while (0); #define PRTE_PMIX_SHOW_HELP "prte.show.help" /* PRTE attribute */ typedef uint16_t prte_attribute_key_t; #define PRTE_ATTR_KEY_T PRTE_UINT16 typedef struct { pmix_list_item_t super; /* required for this to be on lists */ prte_attribute_key_t key; /* key identifier */ bool local; // whether or not to pack/send this value pmix_value_t data; } prte_attribute_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_attribute_t); /* some helper functions */ PRTE_EXPORT pmix_proc_state_t prte_pmix_convert_state(int state); PRTE_EXPORT int prte_pmix_convert_pstate(pmix_proc_state_t); PRTE_EXPORT pmix_status_t prte_pmix_convert_rc(int rc); PRTE_EXPORT int prte_pmix_convert_status(pmix_status_t status); PRTE_EXPORT pmix_status_t prte_pmix_convert_job_state_to_error(int state); PRTE_EXPORT pmix_status_t prte_pmix_convert_proc_state_to_error(int state); PRTE_EXPORT int prte_pmix_register_cleanup(char *path, bool directory, bool ignore, bool jobscope); #ifndef PMIX_DATA_BUFFER_STATIC_INIT #define PMIX_DATA_BUFFER_STATIC_INIT \ { \ .base_ptr = NULL, \ .pack_ptr = NULL, \ .unpack_ptr = NULL, \ .bytes_allocated = 0, \ .bytes_used = 0 \ } #endif #define PRTE_MCA_BASE_VERSION_3_0_0(type, type_major, type_minor, type_release) \ PMIX_MCA_BASE_VERSION_2_1_0("prte", PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, \ PRTE_RELEASE_VERSION, type, type_major, type_minor, type_release) END_C_DECLS #endif prrte-4.1.1~rc2/src/pmix/Makefile.am0000664000175000017500000000234615203542132017444 0ustar alastairalastair# -*- makefile -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2016 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2014-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2015 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This makefile.am does not stand on its own - it is included from src/Makefile.am # Source code files headers += \ pmix/pmix-internal.h libprrte_la_SOURCES += \ pmix/pmix.c prrte-4.1.1~rc2/src/util/0000775000175000017500000000000015203542132015403 5ustar alastairalastairprrte-4.1.1~rc2/src/util/malloc.c0000664000175000017500000001230715203542132017021 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2018 Triad National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include "src/runtime/prte_globals.h" #include "src/util/malloc.h" #include "src/util/pmix_output.h" /* * Undefine "malloc" and "free" */ #if defined(malloc) # undef malloc #endif #if defined(calloc) # undef calloc #endif #if defined(free) # undef free #endif #if defined(realloc) # undef realloc #endif /* * Public variables */ int prte_malloc_debug_level = PRTE_MALLOC_DEBUG_LEVEL; int prte_malloc_output = -1; /* * Private variables */ #if PRTE_ENABLE_DEBUG static pmix_output_stream_t malloc_stream; #endif #if PRTE_ENABLE_DEBUG /* * Finalize the malloc debug interface */ void prte_malloc_finalize(void) { if (-1 != prte_malloc_output) { pmix_output_close(prte_malloc_output); prte_malloc_output = -1; PMIX_DESTRUCT(&malloc_stream); } } /* * Initialize the malloc debug interface */ void prte_malloc_init(void) { PMIX_CONSTRUCT(&malloc_stream, pmix_output_stream_t); malloc_stream.lds_is_debugging = true; malloc_stream.lds_verbose_level = 5; malloc_stream.lds_prefix = "malloc debug: "; malloc_stream.lds_want_stderr = true; prte_malloc_output = pmix_output_open(&malloc_stream); } #else void prte_malloc_init(void) { } void prte_malloc_finalize(void) { } #endif /* PRTE_ENABLE_DEBUG */ /* * Debug version of malloc */ void *prte_malloc(size_t size, const char *file, int line) { void *addr; #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 1) { if (size <= 0) { pmix_output(prte_malloc_output, "Request for %ld bytes (%s, %d)", (long) size, file, line); } } #else PRTE_HIDE_UNUSED_PARAMS(file, line); #endif /* PRTE_ENABLE_DEBUG */ addr = malloc(size); #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 0) { if (NULL == addr) { pmix_output(prte_malloc_output, "Request for %ld bytes failed (%s, %d)", (long) size, file, line); } } #endif /* PRTE_ENABLE_DEBUG */ return addr; } /* * Debug version of calloc */ void *prte_calloc(size_t nmembers, size_t size, const char *file, int line) { void *addr; #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 1) { if (size <= 0) { pmix_output(prte_malloc_output, "Request for %ld zeroed elements of size %ld (%s, %d)", (long) nmembers, (long) size, file, line); } } #else PRTE_HIDE_UNUSED_PARAMS(file, line); #endif /* PRTE_ENABLE_DEBUG */ addr = calloc(nmembers, size); #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 0) { if (NULL == addr) { pmix_output(prte_malloc_output, "Request for %ld zeroed elements of size %ld failed (%s, %d)", (long) nmembers, (long) size, file, line); } } #endif /* PRTE_ENABLE_DEBUG */ return addr; } /* * Debug version of realloc */ void *prte_realloc(void *ptr, size_t size, const char *file, int line) { void *addr; #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 1) { if (size <= 0) { if (NULL == ptr) { pmix_output(prte_malloc_output, "Realloc NULL for %ld bytes (%s, %d)", (long) size, file, line); } else { pmix_output(prte_malloc_output, "Realloc %p for %ld bytes (%s, %d)", ptr, (long) size, file, line); } } } #else PRTE_HIDE_UNUSED_PARAMS(file, line); #endif /* PRTE_ENABLE_DEBUG */ addr = realloc(ptr, size); #if PRTE_ENABLE_DEBUG if (prte_malloc_debug_level > 0) { if (NULL == addr) { pmix_output(prte_malloc_output, "Realloc %p for %ld bytes failed (%s, %d)", ptr, (long) size, file, line); } } #endif /* PRTE_ENABLE_DEBUG */ return addr; } /* * Debug version of free */ void prte_free(void *addr, const char *file, int line) { PRTE_HIDE_UNUSED_PARAMS(file, line); free(addr); } void prte_malloc_debug(int level) { #if PRTE_ENABLE_DEBUG prte_malloc_debug_level = level; #else PRTE_HIDE_UNUSED_PARAMS(level); #endif /* PRTE_ENABLE_DEBUG */ } prrte-4.1.1~rc2/src/util/malloc.h0000664000175000017500000001137715203542132017034 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2018 Triad National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file */ #ifndef PRTE_MALLOC_H #define PRTE_MALLOC_H #include #include /* * THIS FILE CANNOT INCLUDE ANY OTHER PRTE HEADER FILES!!! * * It is included via . Hence, it should not * include ANY other files, nor should it include "prte_config.h". * */ /* * Set PRTE_MALLOC_DEBUG_LEVEL to * 0 for no checking * 1 for basic error checking * 2 for more error checking */ #ifndef PRTE_MALLOC_DEBUG_LEVEL # define PRTE_MALLOC_DEBUG_LEVEL 2 #endif BEGIN_C_DECLS /** * Initialize malloc debug output. * * This function is invoked to setup a dedicated output stream for * malloc debug functions. It does \em not (currently) do anything * other than that (i.e., no internal accounting for tracking * malloc/free statements, etc.). * * It is invoked as part of prte_init(). Although this function is * not \em necessary for PRTE_MALLOC() and PRTE_FREE(), it is strong * recommended because no output messages -- regardless of the * malloc debug level set by prte_malloc_debug() -- will be displayed * unless this function is invoked first. */ PRTE_EXPORT void prte_malloc_init(void); PRTE_EXPORT void prte_malloc_finalize(void); /** * \internal * * Back-end error-checking malloc function for PRTE (you should use * the normal malloc() instead of this function). * * @param size The number of bytes to allocate * @param file Typically the __FILE__ macro * @param line Typically the __LINE__ macro * * This function is only used when --enable-mem-debug was specified to * configure (or by default if you're building in a SVN checkout). */ PRTE_EXPORT void *prte_malloc(size_t size, const char *file, int line) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; /** * \internal * * Back-end error-checking calloc function for PRTE (you should use * the normal calloc() instead of this function). * * @param nmembers Number of elements to malloc * @param size Size of each elements * @param file Typically the __FILE__ macro * @param line Typically the __LINE__ macro * * This function is only used when --enable-mem-debug was specified to * configure (or by default if you're building in a SVN checkout). */ PRTE_EXPORT void *prte_calloc(size_t nmembers, size_t size, const char *file, int line) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; /** * \internal * * Back-end error-checking realloc function for PRTE (you should use * the normal realloc() instead of this function). * * @param ptr Pointer to reallocate * @param size The number of bytes to allocate * @param file Typically the __FILE__ macro * @param line Typically the __LINE__ macro * * This function is only used when --enable-mem-debug was specified to * configure (or by default if you're building in a SVN checkout). */ PRTE_EXPORT void *prte_realloc(void *ptr, size_t size, const char *file, int line) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; /** * \internal * * Back-end error-checking free function for PRTE (you should use * free() instead of this function). * * @param addr Address on the heap to free() * @param file Typically the __FILE__ macro * @param line Typically the __LINE__ macro * * This function is only used when --enable-mem-debug was specified * to configure (or by default if you're building in a SVN * checkout). */ PRTE_EXPORT void prte_free(void *addr, const char *file, int line) __prte_attribute_nonnull__(1); /** * Used to set the debug level for malloc debug. * * @param level The level of debugging (0 = none, 1 = some, 2 = more) * * This value defaults to the PRTE_MALLOC_DEBUG_LEVEL. */ PRTE_EXPORT void prte_malloc_debug(int level); END_C_DECLS #endif /* PRTE_MALLOC_H */ prrte-4.1.1~rc2/src/util/nidmap.h0000664000175000017500000000266215203542132017032 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_NIDMAP_H #define PRTE_NIDMAP_H #include "prte_config.h" #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" /* pass info about the nodes in an allocation */ PRTE_EXPORT int prte_util_nidmap_create(pmix_pointer_array_t *pool, pmix_data_buffer_t *buf); PRTE_EXPORT int prte_util_decode_nidmap(pmix_data_buffer_t *buf); #endif /* PRTE_NIDMAP_H */ prrte-4.1.1~rc2/src/util/stacktrace.c0000664000175000017500000005613715203542132017707 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2017 FUJITSU LIMITED. All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Geoffroy Vallee. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_FCNTL_H # include #else # ifdef HAVE_SYS_FCNTL_H # include # endif #endif #include #include #include "constants.h" #include "src/mca/prtebacktrace/prtebacktrace.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" #include "src/util/error.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/util/stacktrace.h" #ifndef _NSIG # define _NSIG 32 #endif #define HOSTFORMAT "[%s:%05d] " int prte_stacktrace_output_fileno = -1; static char *prte_stacktrace_output_filename_base = NULL; static size_t prte_stacktrace_output_filename_max_len = 0; static char *unable_to_print_msg = "Unable to print stack trace!\n"; /* * Set the stacktrace filename: * stacktrace.PID * -or, if VPID is available- * stacktrace.VPID.PID */ static void set_stacktrace_filename(void) { snprintf(prte_stacktrace_output_filename, prte_stacktrace_output_filename_max_len, "%s.%lu.%lu", prte_stacktrace_output_filename_base, (unsigned long) PRTE_PROC_MY_NAME->rank, (unsigned long) getpid()); return; } /** * This function is being called as a signal-handler in response * to a user-specified signal (e.g. SIGFPE or SIGSEGV). * For Linux/Glibc, it then uses backtrace and backtrace_symbols_fd * to figure the current stack and print that out to stderr. * Where available, the BSD libexecinfo is used to provide Linux/Glibc * compatible backtrace and backtrace_symbols_fd functions. * * @param signo with the signal number raised * @param info with information regarding the reason/send of the signal * @param p * * FIXME: Should distinguish for systems, which don't have siginfo... */ #if PRTE_WANT_PRETTY_PRINT_STACKTRACE static void show_stackframe(int signo, siginfo_t *info, void *p) { char print_buffer[1024]; char *tmp = print_buffer; int size = sizeof(print_buffer); int ret; char *si_code_str = ""; PRTE_HIDE_UNUSED_PARAMS(p); /* Do not print the stack trace */ if (0 > prte_stacktrace_output_fileno && 0 == prte_stacktrace_output_filename_max_len) { /* Raise the signal again, so we don't accidentally mask critical signals. * For critical signals, it is preferred that we call 'raise' instead of * 'exit' or 'abort' so that the return status is set properly for this * process. */ signal(signo, SIG_DFL); raise(signo); return; } /* Update the file name with the RANK, if available */ if (0 < prte_stacktrace_output_filename_max_len) { set_stacktrace_filename(); prte_stacktrace_output_fileno = open(prte_stacktrace_output_filename, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR); if (0 > prte_stacktrace_output_fileno) { pmix_output(0, "Error: Failed to open the stacktrace output file. Default: " "stderr\n\tFilename: %s\n\tErrno: %s", prte_stacktrace_output_filename, strerror(errno)); prte_stacktrace_output_fileno = fileno(stderr); } } /* write out the footer information */ memset(print_buffer, 0, sizeof(print_buffer)); ret = snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT "*** Process received signal ***\n", prte_process_info.nodename, getpid()); if (-1 == write(prte_stacktrace_output_fileno, print_buffer, ret)) { return; } memset(print_buffer, 0, sizeof(print_buffer)); # ifdef HAVE_STRSIGNAL ret = snprintf(tmp, size, HOSTFORMAT "Signal: %s (%d)\n", prte_process_info.nodename, getpid(), strsignal(signo), signo); # else ret = snprintf(tmp, size, HOSTFORMAT "Signal: %d\n", prte_process_info.nodename, getpid(), signo); # endif size -= ret; tmp += ret; if (NULL != info) { switch (signo) { case SIGILL: switch (info->si_code) { # ifdef ILL_ILLOPC case ILL_ILLOPC: si_code_str = "Illegal opcode"; break; # endif # ifdef ILL_ILLOPN case ILL_ILLOPN: si_code_str = "Illegal operand"; break; # endif # ifdef ILL_ILLADR case ILL_ILLADR: si_code_str = "Illegal addressing mode"; break; # endif # ifdef ILL_ILLTRP case ILL_ILLTRP: si_code_str = "Illegal trap"; break; # endif # ifdef ILL_PRVOPC case ILL_PRVOPC: si_code_str = "Privileged opcode"; break; # endif # ifdef ILL_PRVREG case ILL_PRVREG: si_code_str = "Privileged register"; break; # endif # ifdef ILL_COPROC case ILL_COPROC: si_code_str = "Coprocessor error"; break; # endif # ifdef ILL_BADSTK case ILL_BADSTK: si_code_str = "Internal stack error"; break; # endif } break; case SIGFPE: switch (info->si_code) { # ifdef FPE_INTDIV case FPE_INTDIV: si_code_str = "Integer divide-by-zero"; break; # endif # ifdef FPE_INTOVF case FPE_INTOVF: si_code_str = "Integer overflow"; break; # endif case FPE_FLTDIV: si_code_str = "Floating point divide-by-zero"; break; case FPE_FLTOVF: si_code_str = "Floating point overflow"; break; case FPE_FLTUND: si_code_str = "Floating point underflow"; break; # ifdef FPE_FLTRES case FPE_FLTRES: si_code_str = "Floating point inexact result"; break; # endif # ifdef FPE_FLTINV case FPE_FLTINV: si_code_str = "Invalid floating point operation"; break; # endif # ifdef FPE_FLTSUB case FPE_FLTSUB: si_code_str = "Subscript out of range"; break; # endif } break; case SIGSEGV: switch (info->si_code) { # ifdef SEGV_MAPERR case SEGV_MAPERR: si_code_str = "Address not mapped"; break; # endif # ifdef SEGV_ACCERR case SEGV_ACCERR: si_code_str = "Invalid permissions"; break; # endif } break; case SIGBUS: switch (info->si_code) { # ifdef BUS_ADRALN case BUS_ADRALN: si_code_str = "Invalid address alignment"; break; # endif # ifdef BUS_ADRERR case BUS_ADRERR: si_code_str = "Non-existant physical address"; break; # endif # ifdef BUS_OBJERR case BUS_OBJERR: si_code_str = "Object-specific hardware error"; break; # endif } break; case SIGTRAP: switch (info->si_code) { # ifdef TRAP_BRKPT case TRAP_BRKPT: si_code_str = "Process breakpoint"; break; # endif # ifdef TRAP_TRACE case TRAP_TRACE: si_code_str = "Process trace trap"; break; # endif } break; case SIGCHLD: switch (info->si_code) { # ifdef CLD_EXITED case CLD_EXITED: si_code_str = "Child has exited"; break; # endif # ifdef CLD_KILLED case CLD_KILLED: si_code_str = "Child has terminated abnormally and did not create a core file"; break; # endif # ifdef CLD_DUMPED case CLD_DUMPED: si_code_str = "Child has terminated abnormally and created a core file"; break; # endif # ifdef CLD_WTRAPPED case CLD_TRAPPED: si_code_str = "Traced child has trapped"; break; # endif # ifdef CLD_STOPPED case CLD_STOPPED: si_code_str = "Child has stopped"; break; # endif # ifdef CLD_CONTINUED case CLD_CONTINUED: si_code_str = "Stopped child has continued"; break; # endif } break; # ifdef SIGPOLL case SIGPOLL: switch (info->si_code) { # ifdef POLL_IN case POLL_IN: si_code_str = "Data input available"; break; # endif # ifdef POLL_OUT case POLL_OUT: si_code_str = "Output buffers available"; break; # endif # ifdef POLL_MSG case POLL_MSG: si_code_str = "Input message available"; break; # endif # ifdef POLL_ERR case POLL_ERR: si_code_str = "I/O error"; break; # endif # ifdef POLL_PRI case POLL_PRI: si_code_str = "High priority input available"; break; # endif # ifdef POLL_HUP case POLL_HUP: si_code_str = "Device disconnected"; break; # endif } break; # endif /* SIGPOLL */ default: switch (info->si_code) { # ifdef SI_ASYNCNL case SI_ASYNCNL: si_code_str = "SI_ASYNCNL"; break; # endif # ifdef SI_SIGIO case SI_SIGIO: si_code_str = "Queued SIGIO"; break; # endif # ifdef SI_ASYNCIO case SI_ASYNCIO: si_code_str = "Asynchronous I/O request completed"; break; # endif # ifdef SI_MESGQ case SI_MESGQ: si_code_str = "Message queue state changed"; break; # endif case SI_TIMER: si_code_str = "Timer expiration"; break; case SI_QUEUE: si_code_str = "Sigqueue() signal"; break; case SI_USER: si_code_str = "User function (kill, sigsend, abort, etc.)"; break; # ifdef SI_KERNEL case SI_KERNEL: si_code_str = "Kernel signal"; break; # endif /* Dragonfly defines SI_USER and SI_UNDEFINED both as zero: */ /* For some reason, the PGI compiler will not let us combine these two #if tests into a single statement. Sigh. */ # if defined(SI_UNDEFINED) # if SI_UNDEFINED != SI_USER case SI_UNDEFINED: si_code_str = "Undefined code"; break; # endif # endif } } /* print signal errno information */ if (0 != info->si_errno) { ret = snprintf(tmp, size, HOSTFORMAT "Associated errno: %s (%d)\n", prte_process_info.nodename, getpid(), strerror(info->si_errno), info->si_errno); size -= ret; tmp += ret; } ret = snprintf(tmp, size, HOSTFORMAT "Signal code: %s (%d)\n", prte_process_info.nodename, getpid(), si_code_str, info->si_code); size -= ret; tmp += ret; switch (signo) { case SIGILL: case SIGFPE: case SIGSEGV: case SIGBUS: { ret = snprintf(tmp, size, HOSTFORMAT "Failing at address: %p\n", prte_process_info.nodename, getpid(), info->si_addr); size -= ret; tmp += ret; break; } case SIGCHLD: { ret = snprintf(tmp, size, HOSTFORMAT "Sending PID: %d, Sending UID: %d, Status: %d\n", prte_process_info.nodename, getpid(), info->si_pid, info->si_uid, info->si_status); size -= ret; tmp += ret; break; } # ifdef SIGPOLL case SIGPOLL: { # ifdef HAVE_SIGINFO_T_SI_FD ret = snprintf(tmp, size, HOSTFORMAT "Band event: %ld, File Descriptor : %d\n", prte_process_info.nodename, getpid(), (long) info->si_band, info->si_fd); # elif HAVE_SIGINFO_T_SI_BAND ret = snprintf(tmp, size, HOSTFORMAT "Band event: %ld\n", prte_process_info.nodename, getpid(), (long) info->si_band); # else ret = 0; # endif size -= ret; tmp += ret; break; } # endif } } else { ret = snprintf(tmp, size, HOSTFORMAT "siginfo is NULL, additional information unavailable\n", prte_process_info.nodename, getpid()); size -= ret; tmp += ret; } /* write out the signal information generated above */ if (-1 == write(prte_stacktrace_output_fileno, print_buffer, sizeof(print_buffer) - size)) { return; } /* print out the stack trace */ snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT, prte_process_info.nodename, getpid()); ret = prte_backtrace_print(NULL, print_buffer, 2); if (PRTE_SUCCESS != ret) { if (-1 == write(prte_stacktrace_output_fileno, unable_to_print_msg, strlen(unable_to_print_msg))) { return; } } /* write out the footer information */ memset(print_buffer, 0, sizeof(print_buffer)); ret = snprintf(print_buffer, sizeof(print_buffer), HOSTFORMAT "*** End of error message ***\n", prte_process_info.nodename, getpid()); if (ret > 0) { if (-1 == write(prte_stacktrace_output_fileno, print_buffer, ret)) { return; } } else { if (-1 == write(prte_stacktrace_output_fileno, unable_to_print_msg, strlen(unable_to_print_msg))) { return; } } if (fileno(stdout) != prte_stacktrace_output_fileno && fileno(stderr) != prte_stacktrace_output_fileno) { close(prte_stacktrace_output_fileno); prte_stacktrace_output_fileno = -1; } /* Raise the signal again, so we don't accidentally mask critical signals. * For critical signals, it is preferred that we call 'raise' instead of * 'exit' or 'abort' so that the return status is set properly for this * process. */ signal(signo, SIG_DFL); raise(signo); } #endif /* PRTE_WANT_PRETTY_PRINT_STACKTRACE */ #if PRTE_WANT_PRETTY_PRINT_STACKTRACE void prte_stackframe_output(int stream) { int traces_size; char **traces; /* print out the stack trace */ if (PRTE_SUCCESS == prte_backtrace_buffer(&traces, &traces_size)) { int i; /* since we have the opportunity, strip off the bottom two function calls, which will be this function and prte_backtrace_buffer(). */ for (i = 2; i < traces_size; ++i) { pmix_output(stream, "%s", traces[i]); } } else { /* Do not print the stack trace */ if (0 > prte_stacktrace_output_fileno && 0 == prte_stacktrace_output_filename_max_len) { return; } /* Update the file name with the RANK, if available */ if (0 < prte_stacktrace_output_filename_max_len) { set_stacktrace_filename(); prte_stacktrace_output_fileno = open(prte_stacktrace_output_filename, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR); if (0 > prte_stacktrace_output_fileno) { pmix_output(0, "Error: Failed to open the stacktrace output file. Default: " "stderr\n\tFilename: %s\n\tErrno: %s", prte_stacktrace_output_filename, strerror(errno)); prte_stacktrace_output_fileno = fileno(stderr); } } prte_backtrace_print(NULL, NULL, 2); if (fileno(stdout) != prte_stacktrace_output_fileno && fileno(stderr) != prte_stacktrace_output_fileno) { close(prte_stacktrace_output_fileno); prte_stacktrace_output_fileno = -1; } } } char *prte_stackframe_output_string(void) { int traces_size, i; size_t len; char *output, **traces; len = 0; if (PRTE_SUCCESS != prte_backtrace_buffer(&traces, &traces_size)) { return NULL; } /* Calculate the space needed for the string */ for (i = 3; i < traces_size; i++) { if (NULL == traces[i]) { break; } len += strlen(traces[i]) + 1; } output = (char *) malloc(len + 1); if (NULL == output) { return NULL; } *output = '\0'; for (i = 3; i < traces_size; i++) { if (NULL == traces[i]) { break; } strcat(output, traces[i]); strcat(output, "\n"); } free(traces); return output; } #endif /* PRTE_WANT_PRETTY_PRINT_STACKTRACE */ /** * Here we register the show_stackframe function for signals * passed to PRTE by the mpi_signal-parameter passed to mpirun * by the user. * * @returnvalue PRTE_SUCCESS * @returnvalue PRTE_ERR_BAD_PARAM if the value in the signal-list * is not a valid signal-number * */ int prte_util_register_stackhandlers(void) { #if PRTE_WANT_PRETTY_PRINT_STACKTRACE struct sigaction act, old; char *tmp; char *next; bool complain, showed_help = false; /* Setup the output stream to use */ if (NULL == prte_stacktrace_output_filename || 0 == strcasecmp(prte_stacktrace_output_filename, "none")) { prte_stacktrace_output_fileno = -1; } else if (0 == strcasecmp(prte_stacktrace_output_filename, "stdout")) { prte_stacktrace_output_fileno = fileno(stdout); } else if (0 == strcasecmp(prte_stacktrace_output_filename, "stderr")) { prte_stacktrace_output_fileno = fileno(stderr); } else if (0 == strcasecmp(prte_stacktrace_output_filename, "file") || 0 == strcasecmp(prte_stacktrace_output_filename, "file:")) { prte_stacktrace_output_filename_base = strdup("stacktrace"); free(prte_stacktrace_output_filename); // Magic number: 8 = space for .PID and .RANK (allow 7 digits each) prte_stacktrace_output_filename_max_len = strlen("stacktrace") + 8 + 8; prte_stacktrace_output_filename = (char *) malloc( sizeof(char) * prte_stacktrace_output_filename_max_len); set_stacktrace_filename(); prte_stacktrace_output_fileno = -1; } else if (0 == strncasecmp(prte_stacktrace_output_filename, "file:", 5)) { char *filename_cpy = NULL; next = strchr(prte_stacktrace_output_filename, ':'); if (NULL == next) { free(prte_stacktrace_output_filename); free(filename_cpy); return PRTE_ERR_NOT_FOUND; } next++; // move past the ':' to the filename specified prte_stacktrace_output_filename_base = strdup(next); free(prte_stacktrace_output_filename); // Magic number: 8 = space for .PID and .RANK (allow 7 digits each) prte_stacktrace_output_filename_max_len = strlen(prte_stacktrace_output_filename_base) + 8 + 8; prte_stacktrace_output_filename = (char *) malloc( sizeof(char) * prte_stacktrace_output_filename_max_len); set_stacktrace_filename(); prte_stacktrace_output_fileno = -1; free(filename_cpy); } else { prte_stacktrace_output_fileno = fileno(stderr); } /* Setup the signals to catch */ memset(&act, 0, sizeof(act)); act.sa_sigaction = show_stackframe; act.sa_flags = SA_SIGINFO; # ifdef SA_ONESHOT act.sa_flags |= SA_ONESHOT; # else act.sa_flags |= SA_RESETHAND; # endif for (tmp = next = prte_signal_string; next != NULL && *next != '\0'; tmp = next + 1) { int sig; int ret; complain = false; sig = strtol(tmp, &next, 10); /* * If there is no sensible number in the string, exit. * Similarly for any number which is not in the signal-number range */ if (((0 == sig) && (tmp == next)) || (0 > sig) || (_NSIG <= sig)) { pmix_show_help("help-prte-util.txt", "stacktrace bad signal", true, prte_signal_string, tmp); return PRTE_ERR_SILENT; } else if (next == NULL) { return PRTE_ERR_BAD_PARAM; } else if (':' == *next && 0 == strncasecmp(next, ":complain", 9)) { complain = true; next += 9; } else if (',' != *next && '\0' != *next) { return PRTE_ERR_BAD_PARAM; } /* Just query first */ ret = sigaction(sig, NULL, &old); if (0 != ret) { return PRTE_ERR_IN_ERRNO; } /* Was there something already there? */ if (SIG_IGN != old.sa_handler && SIG_DFL != old.sa_handler) { if (!showed_help && complain) { /* JMS This is icky; there is no error message aggregation here so this message may be repeated for every single MPI process... */ pmix_show_help("help-prte-util.txt", "stacktrace signal override", true, sig, sig, sig, prte_signal_string); showed_help = true; } } /* Nope, nothing was there, so put in ours */ else { if (0 != sigaction(sig, &act, NULL)) { return PRTE_ERR_IN_ERRNO; } } } #endif /* PRTE_WANT_PRETTY_PRINT_STACKTRACE */ return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/util/sys_limits.c0000664000175000017500000002130215203542132017744 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 NVIDIA Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #ifdef HAVE_SYS_RESOURCE_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include "constants.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_show_help.h" #include "src/util/sys_limits.h" /* * Create and initialize storage for the system limits */ PRTE_EXPORT prte_sys_limits_t prte_sys_limits = { /* initialized = */ false, /* num_files = */ -1, /* num_procs = */ -1, /* file_size = */ 0}; static int prte_setlimit(int resource, char *value, rlim_t *out) { struct rlimit rlim, rlim_set; rlim_t maxlim; rlim.rlim_cur = 0; if (0 == strcmp(value, "max")) { maxlim = (rlim_t) -1; } else if (0 == strncmp(value, "unlimited", strlen(value))) { maxlim = RLIM_INFINITY; } else { maxlim = strtol(value, NULL, 10); } if (0 <= getrlimit(resource, &rlim)) { if (rlim.rlim_max < maxlim) { rlim_set.rlim_cur = rlim.rlim_max; rlim_set.rlim_max = rlim.rlim_max; } else { rlim_set.rlim_cur = maxlim; rlim_set.rlim_max = maxlim; } if (0 <= setrlimit(resource, &rlim_set)) { rlim.rlim_cur = rlim_set.rlim_cur; } else if (RLIM_INFINITY == maxlim) { /* if unlimited wasn't allowed, try to set * to max allowed */ rlim_set.rlim_cur = rlim.rlim_max; rlim_set.rlim_max = rlim.rlim_max; if (0 <= setrlimit(resource, &rlim_set)) { rlim.rlim_cur = rlim_set.rlim_cur; } else { return PRTE_ERROR; } } else { return PRTE_ERROR; } } else { return PRTE_ERROR; } *out = rlim.rlim_cur; return PRTE_SUCCESS; } int prte_util_init_sys_limits(char **errmsg) { char **lims, **lim = NULL, *setlim; int i, rc = PRTE_ERROR; rlim_t value; /* if limits were not given, then nothing to do */ if (NULL == prte_set_max_sys_limits) { return PRTE_SUCCESS; } /* parse the requested limits to set */ lims = PMIx_Argv_split(prte_set_max_sys_limits, ','); if (NULL == lims) { return PRTE_ERR_OUT_OF_RESOURCE; } /* each limit is expressed as a "param:value" pair */ for (i = 0; NULL != lims[i]; i++) { lim = PMIx_Argv_split(lims[i], ':'); if (1 == PMIx_Argv_count(lim)) { setlim = "max"; } else { setlim = lim[1]; } /* for historical reasons, a value of "1" means * that we set the limits on #files, #children, * and max file size */ if (0 == strcmp(lim[0], "1")) { #if HAVE_DECL_RLIMIT_NOFILE if (PRTE_SUCCESS != prte_setlimit(RLIMIT_NOFILE, "max", &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "openfiles", "max"); goto out; } prte_sys_limits.num_files = value; #endif #if HAVE_DECL_RLIMIT_NPROC if (PRTE_SUCCESS != prte_setlimit(RLIMIT_NPROC, "max", &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "maxchildren", "max"); goto out; } prte_sys_limits.num_procs = value; #endif #if HAVE_DECL_RLIMIT_FSIZE if (PRTE_SUCCESS != prte_setlimit(RLIMIT_FSIZE, "max", &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "filesize", "max"); goto out; } prte_sys_limits.file_size = value; #endif break; } else if (0 == strcmp(lim[0], "0")) { /* user didn't want anything set */ break; } /* process them separately */ if (0 == strcmp(lim[0], "core")) { #if HAVE_DECL_RLIMIT_CORE if (PRTE_SUCCESS != prte_setlimit(RLIMIT_CORE, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "openfiles", setlim); goto out; } #endif } else if (0 == strcmp(lim[0], "filesize")) { #if HAVE_DECL_RLIMIT_FSIZE if (PRTE_SUCCESS != prte_setlimit(RLIMIT_FSIZE, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "filesize", setlim); goto out; } prte_sys_limits.file_size = value; #endif } else if (0 == strcmp(lim[0], "maxmem")) { #if HAVE_DECL_RLIMIT_AS if (PRTE_SUCCESS != prte_setlimit(RLIMIT_AS, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "maxmem", setlim); goto out; } #endif } else if (0 == strcmp(lim[0], "openfiles")) { #if HAVE_DECL_RLIMIT_NOFILE if (PRTE_SUCCESS != prte_setlimit(RLIMIT_NOFILE, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "openfiles", setlim); goto out; } prte_sys_limits.num_files = value; #endif } else if (0 == strcmp(lim[0], "stacksize")) { #if HAVE_DECL_RLIMIT_STACK if (PRTE_SUCCESS != prte_setlimit(RLIMIT_STACK, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "stacksize", setlim); goto out; } #endif } else if (0 == strcmp(lim[0], "maxchildren")) { #if HAVE_DECL_RLIMIT_NPROC if (PRTE_SUCCESS != prte_setlimit(RLIMIT_NPROC, setlim, &value)) { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-failed", true, "maxchildren", setlim); goto out; } prte_sys_limits.num_procs = value; #endif } else { *errmsg = pmix_show_help_string("help-prte-util.txt", "sys-limit-unrecognized", true, lim[0], setlim); goto out; } PMIx_Argv_free(lim); lim = NULL; } /* indicate we initialized the limits structure */ prte_sys_limits.initialized = true; rc = PRTE_SUCCESS; out: PMIx_Argv_free(lims); if (NULL != lim) { PMIx_Argv_free(lim); } return rc; } int prte_getpagesize(void) { static int page_size = -1; if (page_size != -1) { // testing in a loop showed sysconf() took ~5 usec vs ~0.3 usec with it cached return page_size; } #ifdef HAVE_GETPAGESIZE return page_size = getpagesize(); #elif defined(_SC_PAGESIZE) return page_size = sysconf(_SC_PAGESIZE); #elif defined(_SC_PAGE_SIZE) return page_size = sysconf(_SC_PAGE_SIZE); #else return page_size = 65536; /* safer to overestimate than under */ #endif } prrte-4.1.1~rc2/src/util/hostfile/0000775000175000017500000000000015203542132017220 5ustar alastairalastairprrte-4.1.1~rc2/src/util/hostfile/hostfile_lex.l0000664000175000017500000001617115203542132022070 0ustar alastairalastair%option nounput %option noinput %{ /* -*- C -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H #include #endif #include "src/util/hostfile/hostfile_lex.h" /* * local functions */ BEGIN_C_DECLS int prte_util_hostfile_wrap(void); END_C_DECLS int prte_util_hostfile_wrap(void) { prte_util_hostfile_done = true; return 1; } /* * global variables */ int prte_util_hostfile_line=1; prte_hostfile_value_t prte_util_hostfile_value = {0}; bool prte_util_hostfile_done = false; #ifndef YY_BREAK #define YY_BREAK #endif %} WHITE [\f\t\v ] %x comment %% {WHITE}*\n { prte_util_hostfile_line++; return PRTE_HOSTFILE_NEWLINE; } #.*\n { prte_util_hostfile_line++; return PRTE_HOSTFILE_NEWLINE; } "//".*\n { prte_util_hostfile_line++; return PRTE_HOSTFILE_NEWLINE; } "/*" { BEGIN(comment); return PRTE_HOSTFILE_NEWLINE; } [^*\n]* { break; } /* Eat up non '*'s */ "*"+[^*/\n]* { break; } /* Eat '*'s not followed by a '/' */ \n { prte_util_hostfile_line++; return PRTE_HOSTFILE_NEWLINE; } "*"+"/" { BEGIN(INITIAL); /* Done with Block Comment */ return PRTE_HOSTFILE_NEWLINE; } \"[^\"]*\" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_QUOTED_STRING; } {WHITE}+ { break; } /* whitespace */ "=" { return PRTE_HOSTFILE_EQUAL; } cpu { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_CPU; } count { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_COUNT; } slots { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS; } "slots-max" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } slots_max { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } "max-slots" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } max_slots { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } "cpu-max" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } cpu_max { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } "max-cpu" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } max_cpu { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } "count-max" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } count_max { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } "max-count" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } max_count { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOTS_MAX; } rank { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_RANK; } slot { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SLOT; } username { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_USERNAME; } "user-name" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_USERNAME; } "user_name" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_USERNAME; } port { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_PORT; } boards { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_BOARDS; } sockets { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SOCKETS_PER_BOARD; } sockets_per_board { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SOCKETS_PER_BOARD; } "sockets-per-board" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_SOCKETS_PER_BOARD; } cores { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_CORES_PER_SOCKET; } cores_per_socket { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_CORES_PER_SOCKET; } "cores-per-socket" { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_CORES_PER_SOCKET; } \+n[0-9]+ { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_RELATIVE; } \+[eE][\:][0-9]+ { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_RELATIVE; } \+[eE] { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_RELATIVE; } [0-9]+ { prte_util_hostfile_value.ival = atol(yytext); return PRTE_HOSTFILE_INT; } %{ /* First detect hosts as standard Strings (but without ".") * then username@IPv4 or IPV4, then username@IPv6 or IPv6, * followed by username@hostname or hostname */ %} [A-Za-z0-9_\-,:*@]* { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_STRING; } ([A-Za-z0-9][A-Za-z0-9_\-]*"@")?([0-9]{1,3}"."){3}[0-9]{1,3} { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_IPV4; } ([A-Za-z0-9][A-Za-z0-9_\-]*"@")?([A-Fa-f0-9]{0,4}":")+[":"]*([A-Fa-f0-9]{0,4}":")+[A-Fa-f0-9]{1,4} { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_IPV6; } (\^?[A-Za-z0-9][A-Za-z0-9_\-]*"@")?[A-Za-z0-9][A-Za-z0-9_\-\.]* { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_HOSTNAME; } . { prte_util_hostfile_value.sval = yytext; return PRTE_HOSTFILE_ERROR; } %% prrte-4.1.1~rc2/src/util/hostfile/hostfile.c0000664000175000017500000011774515203542132021220 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_UNISTD_H # include #endif #include #include #include #include "src/class/pmix_list.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/prteinstalldirs/prteinstalldirs.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ras/base/base.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/util/hostfile/hostfile.h" #include "src/util/hostfile/hostfile_lex.h" static const char *cur_hostfile_name = NULL; static void hostfile_parse_error(int token) { switch (token) { case PRTE_HOSTFILE_STRING: pmix_show_help("help-hostfile.txt", "parse_error_string", true, cur_hostfile_name, prte_util_hostfile_line, token, prte_util_hostfile_value.sval); break; case PRTE_HOSTFILE_IPV4: case PRTE_HOSTFILE_IPV6: case PRTE_HOSTFILE_INT: pmix_show_help("help-hostfile.txt", "parse_error_int", true, cur_hostfile_name, prte_util_hostfile_line, token, prte_util_hostfile_value.ival); break; default: pmix_show_help("help-hostfile.txt", "parse_error", true, cur_hostfile_name, prte_util_hostfile_line, token); break; } } /** * Return the integer following an = (actually may only return positive ints) */ static int hostfile_parse_int(void) { if (PRTE_HOSTFILE_EQUAL != prte_util_hostfile_lex()) return -1; if (PRTE_HOSTFILE_INT != prte_util_hostfile_lex()) return -1; return prte_util_hostfile_value.ival; } /** * Return the string following an = (option to a keyword) */ static char *hostfile_parse_string(void) { int rc; if (PRTE_HOSTFILE_EQUAL != prte_util_hostfile_lex()) { return NULL; } rc = prte_util_hostfile_lex(); if (PRTE_HOSTFILE_STRING != rc) { return NULL; } return strdup(prte_util_hostfile_value.sval); } static int hostfile_parse_line(int token, pmix_list_t *updates, pmix_list_t *exclude, bool keep_all) { int rc; prte_node_t *node; bool got_max = false; char *value; char **argv; char *node_name = NULL; char *username = NULL; int cnt; int number_of_slots = 0; char buff[64]; char *alias = NULL; if (PRTE_HOSTFILE_STRING == token || PRTE_HOSTFILE_HOSTNAME == token || PRTE_HOSTFILE_INT == token || PRTE_HOSTFILE_IPV4 == token || PRTE_HOSTFILE_IPV6 == token) { if (PRTE_HOSTFILE_INT == token) { snprintf(buff, 64, "%d", prte_util_hostfile_value.ival); value = buff; } else { value = prte_util_hostfile_value.sval; } argv = PMIx_Argv_split(value, '@'); cnt = PMIx_Argv_count(argv); if (1 == cnt) { node_name = strdup(argv[0]); } else if (2 == cnt) { username = strdup(argv[0]); node_name = strdup(argv[1]); } else { pmix_output(0, "WARNING: Unhandled user@host-combination - %s\n", value); /* XXX */ PMIx_Argv_free(argv); return PRTE_ERROR; } PMIx_Argv_free(argv); if (!prte_keep_fqdn_hostnames) { // Strip off the FQDN if present, ignore IP addresses if (!pmix_net_isaddr(node_name)) { char *ptr; alias = strdup(node_name); if (NULL != (ptr = strchr(node_name, '.'))) { *ptr = '\0'; } } } /* if the first letter of the name is '^', then this is a node * to be excluded. Remove the ^ character so the nodename is * usable, and put it on the exclude list */ if ('^' == node_name[0]) { int i, len; len = strlen(node_name); for (i = 1; i < len; i++) { node_name[i - 1] = node_name[i]; } node_name[len - 1] = '\0'; /* truncate */ PMIX_OUTPUT_VERBOSE((3, prte_ras_base_framework.framework_output, "%s hostfile: node %s is being excluded", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node_name)); /* see if this is another name for us */ if (prte_check_host_is_local(node_name)) { /* Nodename has been allocated, that is for sure */ free(node_name); node_name = strdup(prte_process_info.nodename); } /* Do we need to make a new node object? First check to see if it's already in the exclude list */ node = prte_node_match(exclude, node_name); if (NULL == node) { node = PMIX_NEW(prte_node_t); if (prte_keep_fqdn_hostnames || NULL == alias) { node->name = strdup(node_name); } else { node->name = strdup(alias); node->rawname = strdup(node_name); } if (NULL != username) { prte_set_attribute(&node->attributes, PRTE_NODE_USERNAME, PRTE_ATTR_LOCAL, username, PMIX_STRING); } if (NULL != alias && 0 != strcmp(alias, node->name)) { // new node object, so alias must be unique PMIx_Argv_append_nosize(&node->aliases, alias); } pmix_list_append(exclude, &node->super); } else { /* the node name may not match the prior entry, so ensure we * keep it if necessary */ if (0 != strcmp(node_name, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, node_name); } free(node_name); if (NULL != alias && 0 != strcmp(alias, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, alias); } } if (NULL != alias) { free(alias); alias = NULL; } if (NULL != username) { free(username); } return PRTE_SUCCESS; } /* this is not a node to be excluded, so we need to process it and * add it to the "include" list. */ PMIX_OUTPUT_VERBOSE((3, prte_ras_base_framework.framework_output, "%s hostfile: node %s is being included - keep all is %s alias %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node_name, keep_all ? "TRUE" : "FALSE", (NULL == alias) ? "NULL" : alias)); /* see if this is another name for us */ if (prte_check_host_is_local(node_name)) { /* Nodename has been allocated, that is for sure */ free(node_name); node_name = strdup(prte_process_info.nodename); } /* Do we need to make a new node object? */ if (keep_all || NULL == (node = prte_node_match(updates, node_name))) { node = PMIX_NEW(prte_node_t); if (prte_keep_fqdn_hostnames || NULL == alias) { node->name = strdup(node_name); } else { node->name = strdup(node_name); node->rawname = strdup(alias); } free(node_name); node->slots = 1; if (NULL != username) { prte_set_attribute(&node->attributes, PRTE_NODE_USERNAME, PRTE_ATTR_LOCAL, username, PMIX_STRING); } if (NULL != alias && 0 != strcmp(alias, node->name)) { // new node object, so alias must be unique PMIx_Argv_append_nosize(&node->aliases, alias); } pmix_list_append(updates, &node->super); } else { /* this node was already found once - add a slot and mark slots as "given" */ node->slots++; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); /* the node name may not match the prior entry, so ensure we * keep it if necessary */ if (0 != strcmp(node_name, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, node_name); } free(node_name); if (NULL != alias && 0 != strcmp(alias, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, alias); } } if (NULL != alias) { free(alias); alias = NULL; } } else if (PRTE_HOSTFILE_RELATIVE == token) { /* store this for later processing */ node = PMIX_NEW(prte_node_t); // Strip off the FQDN if present, ignore IP addresses if (!pmix_net_isaddr(prte_util_hostfile_value.sval)) { char *ptr; alias = strdup(prte_util_hostfile_value.sval); if (NULL != (ptr = strchr(alias, '.'))) { *ptr = '\0'; } else { free(alias); alias = NULL; } } if (prte_keep_fqdn_hostnames || NULL == alias) { node->name = strdup(prte_util_hostfile_value.sval); } else { node->name = strdup(alias); node->rawname = strdup(prte_util_hostfile_value.sval); } if (NULL != alias && 0 != strcmp(alias, node->name)) { // new node object, so alias must be unique PMIx_Argv_append_nosize(&node->aliases, alias); } pmix_list_append(updates, &node->super); if (NULL != alias) { free(alias); alias = NULL; } } else if (PRTE_HOSTFILE_RANK == token) { /* we can ignore the rank, but we need to extract the node name. we * first need to shift over to the other side of the equal sign as * this is where the node name will be */ while (!prte_util_hostfile_done && PRTE_HOSTFILE_EQUAL != token) { token = prte_util_hostfile_lex(); } if (prte_util_hostfile_done) { /* bad syntax somewhere */ return PRTE_ERROR; } /* next position should be the node name */ token = prte_util_hostfile_lex(); if (PRTE_HOSTFILE_INT == token) { snprintf(buff, 64, "%d", prte_util_hostfile_value.ival); value = buff; } else { value = prte_util_hostfile_value.sval; } argv = PMIx_Argv_split(value, '@'); cnt = PMIx_Argv_count(argv); if (1 == cnt) { node_name = strdup(argv[0]); } else if (2 == cnt) { username = strdup(argv[0]); node_name = strdup(argv[1]); } else { pmix_output(0, "WARNING: Unhandled user@host-combination - %s\n", value); /* XXX */ PMIx_Argv_free(argv); return PRTE_ERROR; } PMIx_Argv_free(argv); // Strip off the FQDN if present, ignore IP addresses if (!prte_keep_fqdn_hostnames && !pmix_net_isaddr(node_name)) { char *ptr; alias = strdup(node_name); if (NULL != (ptr = strchr(alias, '.'))) { *ptr = '\0'; } } /* Do we need to make a new node object? */ if (NULL == (node = prte_node_match(updates, node_name))) { node = PMIX_NEW(prte_node_t); node->name = strdup(node_name); node->slots = 1; if (NULL != username) { prte_set_attribute(&node->attributes, PRTE_NODE_USERNAME, PRTE_ATTR_LOCAL, username, PMIX_STRING); } pmix_list_append(updates, &node->super); } else { /* add a slot */ node->slots++; /* the node name may not match the prior entry, so ensure we * keep it if necessary */ if (0 != strcmp(node_name, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, node_name); } } if (NULL != alias) { PMIx_Argv_append_unique_nosize(&node->aliases, alias); free(alias); node->rawname = strdup(node_name); alias = NULL; } PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s hostfile: node %s slots %d nodes-given %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, node->slots, PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN) ? "TRUE" : "FALSE")); /* mark the slots as "given" since we take them as being the * number specified via the rankfile */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); /* skip to end of line */ while (!prte_util_hostfile_done && PRTE_HOSTFILE_NEWLINE != token) { token = prte_util_hostfile_lex(); } free(node_name); return PRTE_SUCCESS; } else { hostfile_parse_error(token); return PRTE_ERROR; } free(username); while (!prte_util_hostfile_done) { token = prte_util_hostfile_lex(); switch (token) { case PRTE_HOSTFILE_DONE: goto done; case PRTE_HOSTFILE_NEWLINE: goto done; case PRTE_HOSTFILE_USERNAME: username = hostfile_parse_string(); if (NULL != username) { prte_set_attribute(&node->attributes, PRTE_NODE_USERNAME, PRTE_ATTR_LOCAL, username, PMIX_STRING); free(username); } break; case PRTE_HOSTFILE_PORT: rc = hostfile_parse_int(); if (rc < 0) { pmix_show_help("help-hostfile.txt", "port", true, cur_hostfile_name, rc); return PRTE_ERROR; } prte_set_attribute(&node->attributes, PRTE_NODE_PORT, PRTE_ATTR_LOCAL, &rc, PMIX_INT); break; case PRTE_HOSTFILE_COUNT: case PRTE_HOSTFILE_CPU: case PRTE_HOSTFILE_SLOTS: rc = hostfile_parse_int(); if (rc < 0) { pmix_show_help("help-hostfile.txt", "slots", true, cur_hostfile_name, rc); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { /* multiple definitions were given for the * slot count - this is not allowed */ pmix_show_help("help-hostfile.txt", "slots-given", true, cur_hostfile_name, node->name); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } node->slots = rc; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); /* Ensure that slots_max >= slots */ if (node->slots_max != 0 && node->slots_max < node->slots) { node->slots_max = node->slots; } break; case PRTE_HOSTFILE_SLOTS_MAX: rc = hostfile_parse_int(); if (rc < 0) { pmix_show_help("help-hostfile.txt", "max_slots", true, cur_hostfile_name, ((size_t) rc)); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } /* Only take this update if it puts us >= node_slots */ if (rc >= node->slots) { if (node->slots_max != rc) { node->slots_max = rc; got_max = true; } } else { pmix_show_help("help-hostfile.txt", "max_slots_lt", true, cur_hostfile_name, node->slots, rc); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } break; case PRTE_HOSTFILE_STRING: case PRTE_HOSTFILE_INT: /* just ignore it */ break; default: hostfile_parse_error(token); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } if (number_of_slots > node->slots) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); pmix_list_remove_item(updates, &node->super); PMIX_RELEASE(node); return PRTE_ERROR; } } done: if (got_max && !PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { node->slots = node->slots_max; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } return PRTE_SUCCESS; } /** * Parse the specified file into a node list. */ static int hostfile_parse(const char *hostfile, pmix_list_t *updates, pmix_list_t *exclude, bool keep_all) { int token; int rc = PRTE_SUCCESS; cur_hostfile_name = hostfile; prte_util_hostfile_done = false; prte_util_hostfile_in = fopen(hostfile, "r"); if (NULL == prte_util_hostfile_in) { if (NULL == prte_default_hostfile || 0 != strcmp(prte_default_hostfile, hostfile)) { /* not the default hostfile, so not finding it * is an error */ pmix_show_help("help-hostfile.txt", "no-hostfile", true, hostfile); rc = PRTE_ERR_SILENT; goto unlock; } /* if this is the default hostfile and it was given, * then it's an error */ if (prte_default_hostfile_given) { pmix_show_help("help-hostfile.txt", "no-hostfile", true, hostfile); rc = PRTE_ERR_NOT_FOUND; goto unlock; } /* otherwise, not finding it is okay */ rc = PRTE_SUCCESS; goto unlock; } while (!prte_util_hostfile_done) { token = prte_util_hostfile_lex(); switch (token) { case PRTE_HOSTFILE_DONE: prte_util_hostfile_done = true; break; case PRTE_HOSTFILE_NEWLINE: break; /* * This looks odd, since we have several forms of host-definitions: * hostname just plain as it is, being a PRTE_HOSTFILE_STRING * IP4s and user@IPv4s * hostname.domain and user@hostname.domain */ case PRTE_HOSTFILE_STRING: case PRTE_HOSTFILE_INT: case PRTE_HOSTFILE_HOSTNAME: case PRTE_HOSTFILE_IPV4: case PRTE_HOSTFILE_IPV6: case PRTE_HOSTFILE_RELATIVE: case PRTE_HOSTFILE_RANK: rc = hostfile_parse_line(token, updates, exclude, keep_all); if (PRTE_SUCCESS != rc) { goto unlock; } break; default: hostfile_parse_error(token); goto unlock; } } fclose(prte_util_hostfile_in); prte_util_hostfile_in = NULL; prte_util_hostfile_lex_destroy(); unlock: cur_hostfile_name = NULL; return rc; } /** * Parse the provided hostfile and add the nodes to the list. */ int prte_util_add_hostfile_nodes(pmix_list_t *nodes, char *hostfile) { pmix_list_t exclude, adds; pmix_list_item_t *item; int rc, i; prte_node_t *nd, *node; bool found; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s hostfile: checking hostfile %s for nodes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hostfile)); PMIX_CONSTRUCT(&exclude, pmix_list_t); PMIX_CONSTRUCT(&adds, pmix_list_t); /* parse the hostfile and add any new contents to the list */ if (PRTE_SUCCESS != (rc = hostfile_parse(hostfile, &adds, &exclude, false))) { goto cleanup; } /* check for any relative node directives */ PMIX_LIST_FOREACH(node, &adds, prte_node_t) { if ('+' == node->name[0]) { pmix_show_help("help-hostfile.txt", "hostfile:relative-syntax", true, node->name); rc = PRTE_ERR_SILENT; goto cleanup; } } /* remove from the list of nodes those that are in the exclude list */ while (NULL != (item = pmix_list_remove_first(&exclude))) { nd = (prte_node_t *) item; /* check for matches on nodes */ PMIX_LIST_FOREACH(node, &adds, prte_node_t) { if (prte_nptr_match(nd, node)) { /* match - remove it */ pmix_list_remove_item(&adds, &node->super); PMIX_RELEASE(node); break; } } PMIX_RELEASE(item); } /* transfer across all unique nodes */ while (NULL != (item = pmix_list_remove_first(&adds))) { nd = (prte_node_t *) item; found = false; PMIX_LIST_FOREACH(node, nodes, prte_node_t) { if (prte_nptr_match(nd, node)) { found = true; break; } } if (found) { /* add this node name as alias */ PMIx_Argv_append_unique_nosize(&node->aliases, nd->name); /* ensure all other aliases are also transferred */ if (NULL != nd->aliases) { for (i=0; NULL != nd->aliases[i]; i++) { PMIx_Argv_append_unique_nosize(&node->aliases, nd->aliases[i]); } } PMIX_RELEASE(item); } else { pmix_list_append(nodes, &nd->super); PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s hostfile: adding node %s slots %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), nd->name, nd->slots)); } } cleanup: PMIX_LIST_DESTRUCT(&exclude); PMIX_LIST_DESTRUCT(&adds); return rc; } /* Parse the provided hostfile and filter the nodes that are * on the input list, removing those that * are not found in the hostfile */ int prte_util_filter_hostfile_nodes(pmix_list_t *nodes, char *hostfile, bool remove) { pmix_list_t newnodes, exclude; pmix_list_item_t *item1, *item2, *next, *item3; prte_node_t *node_from_list, *node_from_file, *node_from_pool, *node3; int rc = PRTE_SUCCESS; char *cptr; int num_empty, nodeidx; bool want_all_empty = false; pmix_list_t keep; bool found; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s hostfile: filtering nodes through hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hostfile)); /* parse the hostfile and create local list of findings */ PMIX_CONSTRUCT(&newnodes, pmix_list_t); PMIX_CONSTRUCT(&exclude, pmix_list_t); if (PRTE_SUCCESS != (rc = hostfile_parse(hostfile, &newnodes, &exclude, false))) { PMIX_DESTRUCT(&newnodes); PMIX_DESTRUCT(&exclude); return rc; } /* if the hostfile was empty, then treat it as a no-op filter */ if (0 == pmix_list_get_size(&newnodes)) { PMIX_DESTRUCT(&newnodes); PMIX_DESTRUCT(&exclude); /* indicate that the hostfile was empty */ return PRTE_ERR_TAKE_NEXT_OPTION; } /* remove from the list of newnodes those that are in the exclude list * since we could have added duplicate names above due to the */ while (NULL != (item1 = pmix_list_remove_first(&exclude))) { node_from_file = (prte_node_t *) item1; /* check for matches on nodes */ for (item2 = pmix_list_get_first(&newnodes); item2 != pmix_list_get_end(&newnodes); item2 = pmix_list_get_next(item2)) { prte_node_t *node = (prte_node_t *) item2; if (prte_nptr_match(node_from_file, node)) { /* match - remove it */ pmix_list_remove_item(&newnodes, item2); PMIX_RELEASE(item2); break; } } PMIX_RELEASE(item1); } /* now check our nodes and keep or mark those that match. We can * destruct our hostfile list as we go since this won't be needed */ PMIX_CONSTRUCT(&keep, pmix_list_t); while (NULL != (item2 = pmix_list_remove_first(&newnodes))) { node_from_file = (prte_node_t *) item2; next = pmix_list_get_next(item2); /* see if this is a relative node syntax */ if ('+' == node_from_file->name[0]) { /* see if we specified empty nodes */ if ('e' == node_from_file->name[1] || 'E' == node_from_file->name[1]) { /* request for empty nodes - do they want * all of them? */ if (NULL != (cptr = strchr(node_from_file->name, ':'))) { /* the colon indicates a specific # are requested */ cptr++; /* step past : */ num_empty = strtol(cptr, NULL, 10); } else { /* want them all - set num_empty to max */ num_empty = INT_MAX; want_all_empty = true; } /* search the list of nodes provided to us and find those * that are empty */ item1 = pmix_list_get_first(nodes); while (0 < num_empty && item1 != pmix_list_get_end(nodes)) { node_from_list = (prte_node_t *) item1; next = pmix_list_get_next(item1); /* keep our place */ if (0 == node_from_list->slots_inuse) { /* check to see if this node is explicitly called * out later - if so, don't use it here */ for (item3 = pmix_list_get_first(&newnodes); item3 != pmix_list_get_end(&newnodes); item3 = pmix_list_get_next(item3)) { node3 = (prte_node_t *) item3; if (prte_nptr_match(node3, node_from_list)) { /* match - don't use it */ goto skipnode; } } if (remove) { /* remove item from list */ pmix_list_remove_item(nodes, item1); /* xfer to keep list */ pmix_list_append(&keep, item1); } else { /* mark as included */ PRTE_FLAG_SET(node_from_list, PRTE_NODE_FLAG_MAPPED); } --num_empty; } skipnode: item1 = next; } /* did they get everything they wanted? */ if (!want_all_empty && 0 < num_empty) { pmix_show_help("help-hostfile.txt", "hostfile:not-enough-empty", true, num_empty); rc = PRTE_ERR_SILENT; goto cleanup; } } else if ('n' == node_from_file->name[1] || 'N' == node_from_file->name[1]) { /* they want a specific relative node #, so * look it up on global pool */ nodeidx = strtol(&node_from_file->name[2], NULL, 10); node_from_pool = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, nodeidx); if (NULL == node_from_pool) { /* this is an error */ pmix_show_help("help-hostfile.txt", "hostfile:relative-node-not-found", true, nodeidx, node_from_file->name); rc = PRTE_ERR_SILENT; goto cleanup; } /* search the list of nodes provided to us and find it */ for (item1 = pmix_list_get_first(nodes); item1 != pmix_list_get_end(nodes); item1 = pmix_list_get_next(item1)) { node_from_list = (prte_node_t *) item1; if (prte_nptr_match(node_from_pool, node_from_list)) { if (remove) { /* match - remove item from list */ pmix_list_remove_item(nodes, item1); /* xfer to keep list */ pmix_list_append(&keep, item1); } else { /* mark as included */ PRTE_FLAG_SET(node_from_list, PRTE_NODE_FLAG_MAPPED); } break; } } } else { /* invalid relative node syntax */ pmix_show_help("help-hostfile.txt", "hostfile:invalid-relative-node-syntax", true, node_from_file->name); rc = PRTE_ERR_SILENT; goto cleanup; } } else { /* we are looking for a specific node on the list * search the provided list of nodes to see if this * one is found */ found = false; for (item1 = pmix_list_get_first(nodes); item1 != pmix_list_get_end(nodes); item1 = pmix_list_get_next(item1)) { node_from_list = (prte_node_t *) item1; /* we have converted all aliases for ourself * to our own detected nodename */ if (prte_nptr_match(node_from_file, node_from_list)) { /* if the slot count here is less than the * total slots avail on this node, set it * to the specified count - this allows people * to subdivide an allocation */ if (PRTE_FLAG_TEST(node_from_file, PRTE_NODE_FLAG_SLOTS_GIVEN) && node_from_file->slots < node_from_list->slots) { node_from_list->slots = node_from_file->slots; } if (remove) { /* remove the node from the list */ pmix_list_remove_item(nodes, item1); /* xfer it to keep list */ pmix_list_append(&keep, item1); } else { /* mark as included */ PRTE_FLAG_SET(node_from_list, PRTE_NODE_FLAG_MAPPED); } found = true; break; } } /* if the host in the newnode list wasn't found, * then that is an error we need to report to the * user and abort */ if (!found) { pmix_show_help("help-hostfile.txt", "hostfile:extra-node-not-found", true, hostfile, node_from_file->name); rc = PRTE_ERR_SILENT; goto cleanup; } } /* cleanup the newnode list */ PMIX_RELEASE(item2); } /* if we still have entries on our hostfile list, then * there were requested hosts that were not in our allocation. * This is an error - report it to the user and return an error */ if (0 != pmix_list_get_size(&newnodes)) { pmix_show_help("help-hostfile.txt", "not-all-mapped-alloc", true, hostfile); while (NULL != (item1 = pmix_list_remove_first(&newnodes))) { PMIX_RELEASE(item1); } PMIX_DESTRUCT(&newnodes); return PRTE_ERR_SILENT; } if (!remove) { /* all done */ PMIX_DESTRUCT(&newnodes); return PRTE_SUCCESS; } /* clear the rest of the nodes list */ while (NULL != (item1 = pmix_list_remove_first(nodes))) { PMIX_RELEASE(item1); } /* the nodes list has been cleared - rebuild it in order */ while (NULL != (item1 = pmix_list_remove_first(&keep))) { pmix_list_append(nodes, item1); } cleanup: PMIX_DESTRUCT(&newnodes); return rc; } int prte_util_get_ordered_host_list(pmix_list_t *nodes, char *hostfile) { pmix_list_t exclude; pmix_list_item_t *item, *itm, *item2, *item1; char *cptr; int num_empty, i, nodeidx, startempty = 0; bool want_all_empty = false; prte_node_t *node_from_pool, *newnode; int rc; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s hostfile: creating ordered list of hosts from hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hostfile)); PMIX_CONSTRUCT(&exclude, pmix_list_t); /* parse the hostfile and add the contents to the list, keeping duplicates */ if (PRTE_SUCCESS != (rc = hostfile_parse(hostfile, nodes, &exclude, true))) { goto cleanup; } /* parse the nodes to process any relative node directives */ item2 = pmix_list_get_first(nodes); while (item2 != pmix_list_get_end(nodes)) { prte_node_t *node = (prte_node_t *) item2; /* save the next location in case this one gets removed */ item1 = pmix_list_get_next(item2); if ('+' != node->name[0]) { item2 = item1; continue; } /* see if we specified empty nodes */ if ('e' == node->name[1] || 'E' == node->name[1]) { /* request for empty nodes - do they want * all of them? */ if (NULL != (cptr = strchr(node->name, ':'))) { /* the colon indicates a specific # are requested */ cptr++; /* step past : */ num_empty = strtol(cptr, NULL, 10); } else { /* want them all - set num_empty to max */ num_empty = INT_MAX; want_all_empty = true; } /* insert empty nodes into newnodes list in place of the current item. * since item1 is the next item, we insert in front of it */ if (!prte_hnp_is_allocated && 0 == startempty) { startempty = 1; } for (i = startempty; 0 < num_empty && i < prte_node_pool->size; i++) { node_from_pool = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, i); if (NULL == node_from_pool) { continue; } if (0 == node_from_pool->slots_inuse) { newnode = PMIX_NEW(prte_node_t); newnode->name = strdup(node_from_pool->name); /* if the slot count here is less than the * total slots avail on this node, set it * to the specified count - this allows people * to subdivide an allocation */ if (node->slots < node_from_pool->slots) { newnode->slots = node->slots; } else { newnode->slots = node_from_pool->slots; } pmix_list_insert_pos(nodes, item1, &newnode->super); /* track number added */ --num_empty; } } /* bookmark where we stopped in case they ask for more */ startempty = i; /* did they get everything they wanted? */ if (!want_all_empty && 0 < num_empty) { pmix_show_help("help-hostfile.txt", "hostfile:not-enough-empty", true, num_empty); rc = PRTE_ERR_SILENT; goto cleanup; } /* since we have expanded the provided node, remove * it from list */ pmix_list_remove_item(nodes, item2); PMIX_RELEASE(item2); } else if ('n' == node->name[1] || 'N' == node->name[1]) { /* they want a specific relative node #, so * look it up on global pool */ nodeidx = strtol(&node->name[2], NULL, 10); /* if the HNP is not allocated, then we need to * adjust the index as the node pool is offset * by one */ if (!prte_hnp_is_allocated) { nodeidx++; } /* see if that location is filled */ node_from_pool = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, nodeidx); if (NULL == node_from_pool) { /* this is an error */ pmix_show_help("help-hostfile.txt", "hostfile:relative-node-not-found", true, nodeidx, node->name); rc = PRTE_ERR_SILENT; goto cleanup; } /* create the node object */ newnode = PMIX_NEW(prte_node_t); newnode->name = strdup(node_from_pool->name); /* if the slot count here is less than the * total slots avail on this node, set it * to the specified count - this allows people * to subdivide an allocation */ if (node->slots < node_from_pool->slots) { newnode->slots = node->slots; } else { newnode->slots = node_from_pool->slots; } /* insert it before item1 */ pmix_list_insert_pos(nodes, item1, &newnode->super); /* since we have expanded the provided node, remove * it from list */ pmix_list_remove_item(nodes, item2); PMIX_RELEASE(item2); } else { /* invalid relative node syntax */ pmix_show_help("help-hostfile.txt", "hostfile:invalid-relative-node-syntax", true, node->name); rc = PRTE_ERR_SILENT; goto cleanup; } /* move to next */ item2 = item1; } /* remove from the list of nodes those that are in the exclude list */ while (NULL != (item = pmix_list_remove_first(&exclude))) { prte_node_t *exnode = (prte_node_t *) item; /* check for matches on nodes */ for (itm = pmix_list_get_first(nodes); itm != pmix_list_get_end(nodes); itm = pmix_list_get_next(itm)) { prte_node_t *node = (prte_node_t *) itm; if (prte_nptr_match(exnode, node)) { /* match - remove it */ pmix_list_remove_item(nodes, itm); PMIX_RELEASE(itm); /* have to cycle through the entire list as we could * have duplicates */ } } PMIX_RELEASE(item); } cleanup: PMIX_DESTRUCT(&exclude); return rc; } prrte-4.1.1~rc2/src/util/hostfile/hostfile.h0000664000175000017500000000256415203542132021215 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Discovery (Hostfile) */ #ifndef PRTE_UTIL_HOSTFILE_H #define PRTE_UTIL_HOSTFILE_H #include "prte_config.h" #include "src/class/pmix_list.h" BEGIN_C_DECLS PRTE_EXPORT int prte_util_add_hostfile_nodes(pmix_list_t *nodes, char *hostfile); PRTE_EXPORT int prte_util_filter_hostfile_nodes(pmix_list_t *nodes, char *hostfile, bool remove); PRTE_EXPORT int prte_util_get_ordered_host_list(pmix_list_t *nodes, char *hostfile); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/hostfile/help-hostfile.txt0000664000175000017500000003012715203542132022527 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2025-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's dash-host. # [hostfile] PRRTE supports several levels of user-specified hostfiles based on an established precedence order. Users can specify a hostfile that contains a list of nodes to be used for the job, or can provide a comma-delimited list of nodes to be used for that job via the "--host" command line option. The precedence order applied to these various options depends to some extent on the local environment. The following table illustrates how host and hostfile directives work together to define the set of hosts upon which a DVM will execute the job in the absence of a resource manager (RM): +---------+------------+-----------------------------------------------+ | host | hostfile | Result | |=========|============|===============================================| | unset | unset | The DVM will utilize all its available | | | | resources when mapping the job. | +---------+------------+-----------------------------------------------+ | set | unset | Host option defines resource list for the job | +---------+------------+-----------------------------------------------+ | unset | set | Hostfile defines resource list for the job | +---------+------------+-----------------------------------------------+ | set | set | Hostfile defines resource list for the job, | | | | then host filters the list to define the | | | | final set of nodes to be used for the job | +---------+------------+-----------------------------------------------+ Hostfiles (sometimes called "machine files") are a combination of two things: 1. A listing of hosts on which to launch processes. 2. Optionally, limit the number of processes which can be launched on each host. Hostfile syntax consists of one node name on each line, optionally including a designated number of "slots": # This is a comment line, and will be ignored node01 slots=10 node13 slots=5 node15 node16 node17 slots=3 ... Blank lines and lines beginning with a "#" are ignored. A "slot" is the PRRTE term for an allocatable unit where we can launch a process. See the section on definition of the term "slot" for a longer description of slots. In the absence of the "slot" parameter, PRRTE will assign either the number of slots to be the number of CPUs detected on the node or the resource manager-assigned value if operating in the presence of an RM. Important: If using a resource manager, the user-specified number of slots is capped by the RM-assigned value. Relative host indexing ---------------------- Hostfile and "--host" specifications can also be made using relative indexing. This allows a user to stipulate which hosts are to be used for a given app context without specifying the particular host name, but rather its relative position in the allocation. This can probably best be understood through consideration of a few examples. Consider the case where a DVM is comprised of a set of nodes named "foo1", "foo2", "foo3", "foo4". The user wants the first app context to have exclusive use of the first two nodes, and a second app context to use the last two nodes. Of course, the user could printout the allocation to find the names of the nodes allocated to them and then use "--host" to specify this layout, but this is cumbersome and would require hand-manipulation for every invocation. A simpler method is to utilize PRRTE's relative indexing capability to specify the desired layout. In this case, a command line containing: --host +n1,+n2 ./app1 : --host +n3,+n4 ./app2 would provide the desired pattern. The "+" syntax indicates that the information is being provided as a relative index into the existing allocation. Two methods of relative indexing are supported: * "+n#": A relative index into the allocation referencing the "#" node. PRRTE will substitute the "#" node in the allocation * "+e[:#]": A request for "#" empty nodes — i.e., PRRTE is to substitute this reference with nodes that have not yet been used by any other app_context. If the ":#" is not provided, PRRTE will substitute the reference with all empty nodes. Note that PRRTE does track the empty nodes that have been assigned in this manner, so multiple uses of this option will result in assignment of unique nodes up to the limit of the available empty nodes. Requests for more empty nodes than are available will generate an error. Relative indexing can be combined with absolute naming of hosts in any arbitrary manner, and can be used in hostfiles as well as with the "-- host" command line option. In addition, any slot specification provided in hostfiles will be respected — thus, a user can specify that only a certain number of slots from a relative indexed host are to be used for a given app context. Another example may help illustrate this point. Consider the case where the user has a hostfile containing: dummy1 slots=4 dummy2 slots=4 dummy3 slots=4 dummy4 slots=4 dummy5 slots=4 This may, for example, be a hostfile that describes a set of commonly- used resources that the user wishes to execute applications against. For this particular application, the user plans to map byslot, and wants the first two ranks to be on the second node of any allocation, the next ranks to land on an empty node, have one rank specifically on "dummy4", the next rank to be on the second node of the allocation again, and finally any remaining ranks to be on whatever empty nodes are left. To accomplish this, the user provides a hostfile of: +n2 slots=2 +e:1 dummy4 slots=1 +n2 +e The user can now use this information in combination with PRRTE's sequential mapper to obtain their specific layout: --hostfile dummyhosts --hostfile mylayout --map-by seq ./my_app which will result in: rank0 being mapped to dummy3 rank1 to dummy1 as the first empty node rank2 to dummy4 rank3 to dummy3 rank4 to dummy2 and rank5 to dummy5 as the last remaining unused nodes Note that the sequential mapper ignores the number of slots arguments as it only maps one rank at a time to each node in the list. If the default round-robin mapper had been used, then the mapping would have resulted in: * ranks 0 and 1 being mapped to dummy3 since two slots were specified * ranks 2-5 on dummy1 as the first empty node, which has four slots * rank6 on dummy4 since the hostfile specifies only a single slot from that node is to be used * ranks 7 and 8 on dummy3 since only two slots remain available * ranks 9-12 on dummy2 since it is the next available empty node and has four slots * ranks 13-16 on dummy5 since it is the last remaining unused node and has four slots Thus, the use of relative indexing can allow for complex mappings to be ported across allocations, including those obtained from automated resource managers, without the need for manual manipulation of scripts and/or command lines. See the "Host specification" HTML documentation for details about the format and content of hostfiles. # [machinefile] This option is a synonym for "--hostfile". #include#help-hostfile.txt#hostfile # [host] Host syntax consists of a comma-delimited list of node names, each entry optionally containing a ":N" extension indicating the number of slots to assign to that entry: --host node01:5,node02 In the absence of the slot extension, one slot will be assigned to the node. Duplicate entries are aggregated and the number of slots assigned to that node are summed together. Note: A "slot" is the PRRTE term for an allocatable unit where we can launch a process. Thus, the number of slots equates to the maximum number of processes PRRTE may start on that node without oversubscribing it. # [add-hostfile] PRRTE allows a user to expand an existing DVM prior to launching an application. Users can specify a hostfile that contains a list of nodes to be added to the DVM using normal hostfile syntax. The list can include nodes that are already part of the DVM — in this case, the number of slots available on those nodes will be set to the new specification, or adjusted as directed: node01 slots=5 would direct that node01 be set to 5 slots, while node01 slots+=5 would add 5 slots to the current value for node01, and node01 slots-=5 would subtract 5 slots from the current value. Slot adjustments for existing nodes will have no impact on currently executing jobs, but will be applied to any new spawn requests. Nodes contained in the add-hostfile specification are available for immediate use by the accompanying application. Users desiring to constrain the accompanying application to the newly added nodes should also include the "--hostfile" command line directive, giving the same hostfile as its argument: --add-hostfile --hostfile # [no-hostfile] PRTE was unable to open the hostfile: %s Check to make sure the path and filename are correct. # [port] PRTE detected a bad parameter in the hostfile: %s The port parameter is less than 0: port=%d # [slots] PRTE detected a bad parameter in the hostfile: %s The slots parameter is less than 0: slots=%d # [max_slots] PRTE detected a bad parameter in the hostfile: %s The max_slots parameter is less than 0: max_slots=%d # [max_slots_lt] PRTE detected a bad parameter in the hostfile: %s The max_slots parameter is less than the slots parameter: slots=%d max_slots=%d # [parse_error_string] PRTE detected a parse error in the hostfile: %s It occured on line number %d on token %d: %s # [parse_error_int] PRTE detected a parse error in the hostfile: %s It occured on line number %d on token %d: %d # [parse_error] PRTE detected a parse error in the hostfile: %s It occured on line number %d on token %d. # [not-all-mapped-alloc] Some of the requested hosts are not included in the current allocation. The requested hosts were in this hostfile: %s Please verify that you have specified the allocated resources properly in the provided hostfile. # [hostfile:relative-syntax] A relative host was specified, but no prior allocation has been made. Thus, there is no way to determine the proper host to be used. hostfile entry: %s Re-run this command with "--help hosts" for further information. # [hostfile:relative-node-not-found] A relative host was specified, but was not found. The requested host was specified as: Index: %d Syntax given: %s This is most likely due to the relative index being out of bounds. You could obtain a larger allocation or reduce the relative host index. Re-run this command with "--help hosts" for further information. [hostfile:invalid-relative-node-syntax] A relative host was improperly specified — the value provided was. hostfile entry: %s You may have forgotten to preface a node with "N" or "n", or used the "e" or "E" to indicate empty nodes. Re-run this command with "--help hosts" for further information. # [hostfile:not-enough-empty] The requested number of empty hosts was not available — the system was short by %d hosts. Please recheck your allocation. Re-run this command with "--help hosts" for further information. # [hostfile:extra-node-not-found] A hostfile was provided that contains at least one node not present in the allocation: hostfile: %s node: %s If you are operating in a resource-managed environment, then only nodes that are in the allocation can be used in the hostfile. You may find relative node syntax to be a useful alternative to specifying absolute node names; re-run this command with "--help hosts" for further information. # [slots-given] A hostfile was provided that contains multiple definitions of the slot count for at least one node: hostfile: %s node: %s You can either list a node multiple times, once for each slot, or you can provide a single line that contains "slot=N". Mixing the two methods is not supported. Please correct the hostfile and try again. prrte-4.1.1~rc2/src/util/hostfile/hostfile_lex.h0000664000175000017500000000545015203542132022062 0ustar alastairalastair/* -*- C -*- * * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_UTIL_HOSTFILE_LEX_H_ #define PRTE_UTIL_HOSTFILE_LEX_H_ #include "prte_config.h" #ifdef malloc # undef malloc #endif #ifdef realloc # undef realloc #endif #ifdef free # undef free #endif #include typedef union { int ival; char *sval; } prte_hostfile_value_t; extern int prte_util_hostfile_lex(void); extern FILE *prte_util_hostfile_in; extern int prte_util_hostfile_line; extern bool prte_util_hostfile_done; extern prte_hostfile_value_t prte_util_hostfile_value; extern int prte_util_hostfile_lex_destroy(void); /* * Make lex-generated files not issue compiler warnings */ #define YY_STACK_USED 0 #define YY_ALWAYS_INTERACTIVE 0 #define YY_NEVER_INTERACTIVE 0 #define YY_MAIN 0 #define YY_NO_UNPUT 1 #define YY_SKIP_YYWRAP 1 #define PRTE_HOSTFILE_DONE 0 #define PRTE_HOSTFILE_ERROR 1 #define PRTE_HOSTFILE_QUOTED_STRING 2 #define PRTE_HOSTFILE_EQUAL 3 #define PRTE_HOSTFILE_INT 4 #define PRTE_HOSTFILE_STRING 5 #define PRTE_HOSTFILE_CPU 6 #define PRTE_HOSTFILE_COUNT 7 #define PRTE_HOSTFILE_SLOTS 8 #define PRTE_HOSTFILE_SLOTS_MAX 9 #define PRTE_HOSTFILE_USERNAME 10 #define PRTE_HOSTFILE_IPV4 11 #define PRTE_HOSTFILE_HOSTNAME 12 #define PRTE_HOSTFILE_NEWLINE 13 #define PRTE_HOSTFILE_IPV6 14 #define PRTE_HOSTFILE_SLOT 15 #define PRTE_HOSTFILE_RELATIVE 16 #define PRTE_HOSTFILE_BOARDS 17 #define PRTE_HOSTFILE_SOCKETS_PER_BOARD 18 #define PRTE_HOSTFILE_CORES_PER_SOCKET 19 /* ensure we can handle a rank_file input */ #define PRTE_HOSTFILE_RANK 20 #define PRTE_HOSTFILE_PORT 21 #endif prrte-4.1.1~rc2/src/util/hostfile/Makefile.am0000664000175000017500000000263215203542132021257 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2022-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AM_LFLAGS = -Pprte_util_hostfile_ LEX_OUTPUT_ROOT = lex.prte_util_hostfile_ # we do NOT want picky compilers down here CFLAGS = $(PRTE_CFLAGS_BEFORE_PICKY) noinst_LTLIBRARIES = libprrteutilhostfile.la # Source code files libprrteutilhostfile_la_SOURCES = \ hostfile_lex.h \ hostfile_lex.l \ hostfile.h \ hostfile.c maintainer-clean-local: rm -f hostfile_lex.c prrte-4.1.1~rc2/src/util/prte_cmd_line.h0000664000175000017500000003131415203542132020362 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2022 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_CMD_LINE_H #define PRTE_CMD_LINE_H #include "prte_config.h" #ifdef HAVE_UNISTD_H # include #endif #include #include #include #include #include "src/class/pmix_list.h" #include "src/class/pmix_object.h" #include "src/util/pmix_argv.h" BEGIN_C_DECLS /* define the command line options that PRRTE internally understands. * It is the responsibility of each schizo component to translate its * command line inputs to these definitions. The definitions are provided * to help avoid errors due to typos - i.e., where the schizo component * interprets its CLI but assigns it to an erroneous string */ // NAME STRING ARGUMENT // Basic options #define PRTE_CLI_HELP "help" // optional #define PRTE_CLI_VERSION "version" // none #define PRTE_CLI_VERBOSE "verbose" // number of instances => verbosity level #define PRTE_CLI_PARSEABLE "parseable" // none #define PRTE_CLI_PARSABLE "parsable" // none #define PRTE_CLI_PERSONALITY "personality" // required // MCA parameter options #define PRTE_CLI_PRTEMCA "prtemca" // requires TWO #define PRTE_CLI_PMIXMCA "pmixmca" // requires TWO #define PRTE_CLI_TUNE "tune" // required // DVM options #define PRTE_CLI_NO_READY_MSG "no-ready-msg" // none #define PRTE_CLI_DAEMONIZE "daemonize" // none #define PRTE_CLI_SYSTEM_SERVER "system-server" // none #define PRTE_CLI_SET_SID "set-sid" // none #define PRTE_CLI_REPORT_PID "report-pid" // required #define PRTE_CLI_REPORT_URI "report-uri" // required #define PRTE_CLI_TEST_SUICIDE "test-suicide" // none #define PRTE_CLI_DEFAULT_HOSTFILE "default-hostfile" // required #define PRTE_CLI_SINGLETON "singleton" // required #define PRTE_CLI_KEEPALIVE "keepalive" // required #define PRTE_CLI_LAUNCH_AGENT "launch-agent" // required #define PRTE_CLI_MAX_VM_SIZE "max-vm-size" // required #define PRTE_CLI_DEBUG "debug" // none #define PRTE_CLI_DEBUG_DAEMONS "debug-daemons" // none #define PRTE_CLI_DEBUG_DAEMONS_FILE "debug-daemons-file" // none #define PRTE_CLI_LEAVE_SESSION_ATTACHED "leave-session-attached" // none #define PRTE_CLI_TMPDIR "tmpdir" // required #define PRTE_CLI_PREFIX "prefix" // required #define PRTE_CLI_PMIX_PREFIX "pmix-prefix" // required #define PRTE_CLI_APP_PREFIX "app-pmix-prefix" // required #define PRTE_CLI_NOPREFIX "noprefix" // none #define PRTE_CLI_NO_APP_PREFIX "no-app-prefix" // none #define PRTE_CLI_FWD_SIGNALS "forward-signals" // required #define PRTE_CLI_RUN_AS_ROOT "allow-run-as-root" // none #define PRTE_CLI_STREAM_BUF "stream-buffering" // required #define PRTE_CLI_BOOTSTRAP "bootstrap" // none #define PRTE_CLI_HOMO_NODES "uniform-nodes" // none // Application options #define PRTE_CLI_NP "np" // required #define PRTE_CLI_NPERNODE "N" // required #define PRTE_CLI_APPFILE "app" // required #define PRTE_CLI_FWD_ENVAR "x" // required #define PRTE_CLI_FWD_ENVIRON "fwd-environment" // optional #define PRTE_CLI_HOSTFILE "hostfile" // required #define PRTE_CLI_ADDHOSTFILE "add-hostfile" // required #define PRTE_CLI_HOST "host" // required #define PRTE_CLI_ADDHOST "add-host" // required #define PRTE_CLI_PATH "path" // required #define PRTE_CLI_PSET "pset" // required #define PRTE_CLI_PRELOAD_FILES "preload-files" // required #define PRTE_CLI_PRELOAD_BIN "preload-binary" // none #define PRTE_CLI_STDIN "stdin" // required #define PRTE_CLI_OUTPUT "output" // required #define PRTE_CLI_WDIR "wdir" // required #define PRTE_CLI_SET_CWD_SESSION "set-cwd-to-session-dir" // none #define PRTE_CLI_ENABLE_RECOVERY "enable-recovery" // none #define PRTE_CLI_DISABLE_RECOVERY "disable-recovery" // none #define PRTE_CLI_MEM_ALLOC_KIND "memory-alloc-kinds" // required #define PRTE_CLI_GPU_SUPPORT "gpu-support" // required // Placement options #define PRTE_CLI_MAPBY "mapby" // required #define PRTE_CLI_RANKBY "rankby" // required #define PRTE_CLI_BINDTO "bindto" // required // Runtime options #define PRTE_CLI_RTOS "rtos" // required // Debug options #define PRTE_CLI_DO_NOT_LAUNCH "do-not-launch" // none #define PRTE_CLI_DISPLAY "display" // required #define PRTE_CLI_XTERM "xterm" // none #define PRTE_CLI_DO_NOT_AGG_HELP "no-aggregate-help" // none // Tool connection options #define PRTE_CLI_SYS_SERVER_FIRST "system-server-first" // none #define PRTE_CLI_SYS_SERVER_ONLY "system-server-only" // none #define PRTE_CLI_DO_NOT_CONNECT "do-not-connect" // none #define PRTE_CLI_WAIT_TO_CONNECT "wait-to-connect" // required #define PRTE_CLI_NUM_CONNECT_RETRIES "num-connect-retries" // required #define PRTE_CLI_PID "pid" // required #define PRTE_CLI_NAMESPACE "namespace" // required #define PRTE_CLI_DVM_URI "dvm-uri" // required #define PRTE_CLI_DVM "dvm" // optional // Daemon-specific CLI options #define PRTE_CLI_PUBSUB_SERVER "prte-server" // required #define PRTE_CLI_CONTROLLER_URI "dvm-master-uri" // required #define PRTE_CLI_PARENT_URI "parent-uri" // required #define PRTE_CLI_TREE_SPAWN "tree-spawn" // required #define PRTE_CLI_PLM "plm" // required /* define accepted synonyms - these must be defined on the schizo component's * command line in order to be accepted, but PRRTE will automatically translate * them to their accepted synonym */ #define PRTE_CLI_MACHINEFILE "machinefile" // synonym for "hostfile" #define PRTE_CLI_WD "wd" // synonym for "wdir /* define the command line directives PRRTE recognizes */ // Placement directives - used by mapping and binding #define PRTE_CLI_SLOT "slot" #define PRTE_CLI_HWT "hwthread" #define PRTE_CLI_CORE "core" #define PRTE_CLI_L1CACHE "l1cache" #define PRTE_CLI_L2CACHE "l2cache" #define PRTE_CLI_L3CACHE "l3cache" #define PRTE_CLI_NUMA "numa" #define PRTE_CLI_PACKAGE "package" #define PRTE_CLI_NODE "node" #define PRTE_CLI_SEQ "seq" #define PRTE_CLI_DIST "dist" #define PRTE_CLI_PPR "ppr" #define PRTE_CLI_RANKFILE "rankfile" #define PRTE_CLI_NONE "none" #define PRTE_CLI_HWTCPUS "hwtcpus" #define PRTE_CLI_PELIST "pe-list=" // Ranking directives // PRTE_CLI_SLOT, PRTE_CLI_NODE, PRTE_CLI_SPAN reused here #define PRTE_CLI_FILL "fill" #define PRTE_CLI_OBJ "object" // Output directives #define PRTE_CLI_TAG "tag" #define PRTE_CLI_TAG_DET "tag-detailed" #define PRTE_CLI_TAG_FULL "tag-fullname" #define PRTE_CLI_RANK "rank" #define PRTE_CLI_TIMESTAMP "timestamp" #define PRTE_CLI_XML "xml" #define PRTE_CLI_MERGE_ERROUT "merge-stderr-to-stdout" #define PRTE_CLI_DIR "directory" #define PRTE_CLI_FILE "filename" // Display directives #define PRTE_CLI_ALLOC "allocation" #define PRTE_CLI_MAP "map" #define PRTE_CLI_BIND "bind" #define PRTE_CLI_MAPDEV "map-devel" #define PRTE_CLI_TOPO "topo=" #define PRTE_CLI_CPUS "cpus=" #define PRTE_CLI_PHYSICAL_CPUS "physical" // Runtime directives #define PRTE_CLI_ERROR_NZ "error-nonzero-status" // optional arg #define PRTE_CLI_NOLAUNCH "donotlaunch" // no arg #define PRTE_CLI_NOSPAWN "donotspawn" // no arg #define PRTE_CLI_SHOW_PROGRESS "show-progress" // optional arg #define PRTE_CLI_RECOVERABLE "recoverable" // optional arg #define PRTE_CLI_AUTORESTART "autorestart" // optional arg #define PRTE_CLI_CONTINUOUS "continuous" // optional arg #define PRTE_CLI_MAX_RESTARTS "max-restarts" // reqd arg #define PRTE_CLI_EXEC_AGENT "exec-agent" // reqd arg #define PRTE_CLI_DEFAULT_EXEC_AGENT "default-exec-agent" // no arg #define PRTE_CLI_STOP_ON_EXEC "stop-on-exec" // optional arg #define PRTE_CLI_STOP_IN_INIT "stop-in-init" // optional arg #define PRTE_CLI_STOP_IN_APP "stop-in-app" // optional arg #define PRTE_CLI_TIMEOUT "timeout" // reqd arg #define PRTE_CLI_SPAWN_TIMEOUT "spawn-timeout" // reqd arg #define PRTE_CLI_REPORT_STATE "report-state-on-timeout" // optional arg #define PRTE_CLI_STACK_TRACES "get-stack-traces" // optional arg #define PRTE_CLI_REPORT_CHILD_SEP "report-child-jobs-separately" // optional arg #define PRTE_CLI_AGG_HELP "aggregate-help" // optional arg #define PRTE_CLI_NOTIFY_ERRORS "notifyerrors" // optional flag #define PRTE_CLI_OUTPUT_PROCTABLE "output-proctable" // optional arg /* define the command line qualifiers PRRTE recognizes */ // Placement qualifiers #define PRTE_CLI_PE "pe=" #define PRTE_CLI_SPAN "span" #define PRTE_CLI_OVERSUB "oversubscribe" #define PRTE_CLI_NOOVER "nooversubscribe" #define PRTE_CLI_NOLOCAL "nolocal" // PRTE_CLI_HWTCPUS reused here #define PRTE_CLI_CORECPUS "corecpus" #define PRTE_CLI_DEVICE "device=" #define PRTE_CLI_INHERIT "inherit" #define PRTE_CLI_NOINHERIT "noinherit" #define PRTE_CLI_QDIR "dir=" #define PRTE_CLI_QFILE "file=" #define PRTE_CLI_OVERLOAD "overload-allowed" #define PRTE_CLI_NOOVERLOAD "no-overload" #define PRTE_CLI_IF_SUPP "if-supported" #define PRTE_CLI_ORDERED "ordered" #define PRTE_CLI_REPORT "report" #define PRTE_CLI_DISPALLOC "displayalloc" // PRTE_CLI_DISPLAY reused here #define PRTE_CLI_DISPDEV "displaydevel" #define PRTE_CLI_LIMIT "limit=" // Output qualifiers #define PRTE_CLI_NOCOPY "nocopy" #define PRTE_CLI_RAW "raw" #define PRTE_CLI_PATTERN "pattern" END_C_DECLS #endif /* PRTE_CMD_LINE_H */ prrte-4.1.1~rc2/src/util/parse_options.h0000664000175000017500000000242315203542132020442 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * */ #ifndef _PRTE_PARSE_OPTIONS_H_ #define _PRTE_PARSE_OPTIONS_H_ #include "prte_config.h" BEGIN_C_DECLS PRTE_EXPORT void pmix_util_parse_range_options(char *input, char ***output); PRTE_EXPORT void prte_util_get_ranges(char *inp, char ***startpts, char ***endpts); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/numtostr.h0000664000175000017500000000314315203542132017450 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file */ #ifndef PRTE_NUMTOSTR_UTIL #define PRTE_NUMTOSTR_UTIL #include "prte_config.h" /** * Convert a long integer to a char* string. The returned buffer is * allocated by calling malloc() and must be freed by the caller. * * @param num (IN) Input number * @return String containing number (NULL on failure) */ PRTE_EXPORT char *prte_ltostr(long num); /** * Convert a double to a char* string. The returned buffer is allocated * by calling malloc() and must be freed by the caller. * * @param num (IN) Input number * @return String containing number (NULL on failure) */ PRTE_EXPORT char *prte_dtostr(double num); #endif /* PRTE_NUMTOSTR_UTIL */ prrte-4.1.1~rc2/src/util/help-prte-util.txt0000664000175000017500000000363515203542132021026 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's util directory. # [stacktrace signal override] PRTE was inserting a signal handler for signal %d but noticed that there is already a non-default handler installed. PRTE's handler was therefore not installed; your job will continue. This warning message will only be displayed once, even if PRTE encounters this situation again. To avoid displaying this warning message, you can either not install the error handler for signal %d or you can have PRTE not try to install its own signal handler for this signal by setting the "prte_signals" MCA parameter. Signal: %d Current prte_signal value: %s # [stacktrace bad signal] PRTE was inserting a signal handler but was given an invalid signal number: Signal string: %s Bad value: %s The given value must be an integer within the signal number range. Please correct the value and try again. # [malformed-uri] PRTE has detected a malformed URI: URI: %s Accepted values follow IETF RFC3986, e.g. "file://192.168.1.1/over/there" # [relative-path] When creating a URI, all files must be specified in absolute paths: Value provided: %s Please update your application to provide the full path to the file. # [sys-limit-failed] Per request, PRTE attempted to set a system resource limit to a given value: Resource: %s Limit: %s The system has refused to allow this operation. This is likely due to a permission limitation, or specifying an unsupported value. Please check the system or remove the request and try again. # [sys-limit-unrecognized] PRTE received a request to set a system resource limit. Sadly, OMPI does not recognize or currently support the specified resource: Resource: %s Limit: %s Please correct the request and try again. prrte-4.1.1~rc2/src/util/bit_ops.h0000664000175000017500000001100615203542132017211 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2011 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_BIT_OPS_H #define PRTE_BIT_OPS_H #include "prefetch.h" /** * Calculates the highest bit in an integer * * @param value The integer value to examine * @param start Position to start looking * * @returns pos Position of highest-set integer or -1 if none are set. * * Look at the integer "value" starting at position "start", and move * to the right. Return the index of the highest bit that is set to * 1. * * WARNING: *NO* error checking is performed. This is meant to be a * fast inline function. * Using __builtin_clz (count-leading-zeros) uses 3 cycles instead * of 17 cycles (on average value, with start=32) * compared to the loop-version (on Intel Nehalem -- with icc-12.1.0 -O2). */ static inline int prte_hibit(int value, int start) { unsigned int mask; #if PRTE_C_HAVE_BUILTIN_CLZ /* Only look at the part that the caller wanted looking at */ mask = value & ((1 << start) - 1); if (PMIX_UNLIKELY(0 == mask)) { return -1; } start = (8 * sizeof(int) - 1) - __builtin_clz(mask); #else --start; mask = 1 << start; for (; start >= 0; --start, mask >>= 1) { if (value & mask) { break; } } #endif return start; } /** * Returns the cube dimension of a given value. * * @param value The integer value to examine * * @returns cubedim The smallest cube dimension containing that value * * Look at the integer "value" and calculate the smallest power of two * dimension that contains that value. * * WARNING: *NO* error checking is performed. This is meant to be a * fast inline function. * Using __builtin_clz (count-leading-zeros) uses 3 cycles instead of 50 cycles * compared to the loop-version (on Intel Nehalem -- with icc-12.1.0 -O2). */ static inline int prte_cube_dim(int value) { int dim, size; #if PRTE_C_HAVE_BUILTIN_CLZ if (PMIX_UNLIKELY(1 >= value)) { return 0; } size = 8 * sizeof(int); dim = size - __builtin_clz(value - 1); #else for (dim = 0, size = 1; size < value; ++dim, size <<= 1) /* empty */ ; #endif return dim; } /** * @brief Returns next power-of-two of the given value. * * @param value The integer value to return power of 2 * * @returns The next power of two * * WARNING: *NO* error checking is performed. This is meant to be a * fast inline function. * Using __builtin_clz (count-leading-zeros) uses 4 cycles instead of 77 * compared to the loop-version (on Intel Nehalem -- with icc-12.1.0 -O2). */ static inline int prte_next_poweroftwo(int value) { int power2; #if PRTE_C_HAVE_BUILTIN_CLZ if (PMIX_UNLIKELY(0 == value)) { return 1; } power2 = 1 << (8 * sizeof(int) - __builtin_clz(value)); #else for (power2 = 1; value > 0; value >>= 1, power2 <<= 1) /* empty */ ; #endif return power2; } /** * @brief Returns next power-of-two of the given value (and the value itselve if already * power-of-two). * * @param value The integer value to return power of 2 * * @returns The next power of two (inclusive) * * WARNING: *NO* error checking is performed. This is meant to be a * fast inline function. * Using __builtin_clz (count-leading-zeros) uses 4 cycles instead of 56 * compared to the loop-version (on Intel Nehalem -- with icc-12.1.0 -O2). */ static inline int prte_next_poweroftwo_inclusive(int value) { int power2; #if PRTE_C_HAVE_BUILTIN_CLZ if (PMIX_UNLIKELY(1 >= value)) { return 1; } power2 = 1 << (8 * sizeof(int) - __builtin_clz(value - 1)); #else for (power2 = 1; power2 < value; power2 <<= 1) /* empty */ ; #endif return power2; } #endif /* PRTE_BIT_OPS_H */ prrte-4.1.1~rc2/src/util/nidmap.c0000664000175000017500000003164415203542132017027 0ustar alastairalastair/* * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2020 Triad National Security, LLC. All rights * reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "types.h" #ifdef HAVE_UNISTD_H # include #endif #include #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/base/base.h" #include "src/rml/rml.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/nidmap.h" int prte_util_nidmap_create(pmix_pointer_array_t *pool, pmix_data_buffer_t *buffer) { char *raw = NULL; pmix_rank_t *vpids = NULL; uint8_t u8; int n, m, ndaemons, nbytes; bool compressed; char **names = NULL; char **aliases = NULL, **als; prte_node_t *nptr; pmix_byte_object_t bo; size_t sz; pmix_status_t rc; /* pack a flag indicating if the HNP was included in the allocation */ if (prte_hnp_is_allocated) { u8 = 1; } else { u8 = 0; } rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &u8, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack a flag indicating if we are in a managed allocation */ if (prte_managed_allocation) { u8 = 1; } else { u8 = 0; } rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &u8, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* daemon vpids start from 0 and increase linearly by one * up to the number of nodes in the system. The vpid is * a 32-bit value. We don't know how many of the nodes * in the system have daemons - we may not be using them * all just yet. However, even the largest systems won't * have more than a million nodes for quite some time, * so for now we'll just allocate enough space to hold * them all. Someone can optimize this further later */ nbytes = prte_process_info.num_daemons * sizeof(pmix_rank_t); vpids = (pmix_rank_t *) malloc(nbytes); ndaemons = 0; for (n = 0; n < pool->size; n++) { if (NULL == (nptr = (prte_node_t *) pmix_pointer_array_get_item(pool, n))) { continue; } if (NULL == nptr->daemon) { continue; } /* add the hostname to the argv */ PMIx_Argv_append_nosize(&names, nptr->name); als = NULL; if (NULL != nptr->aliases) { for (m=0; NULL != nptr->aliases[m]; m++) { // skip any localhost entries if (0 == strcmp(nptr->aliases[m], "localhost") || 0 == strcmp(nptr->aliases[m], "127.0.0.1")) { continue; } PMIx_Argv_append_nosize(&als, nptr->aliases[m]); } raw = PMIx_Argv_join(als, ','); PMIx_Argv_free(als); PMIx_Argv_append_nosize(&aliases, raw); free(raw); } else { PMIx_Argv_append_nosize(&aliases, "PRTENONE"); } /* store the vpid */ vpids[ndaemons] = nptr->daemon->name.rank; ++ndaemons; } /* little protection */ if (NULL == names || NULL == aliases) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); free(vpids); return PRTE_ERR_NOT_FOUND; } /* construct the string of node names for compression */ raw = PMIx_Argv_join(names, ','); PMIx_Argv_free(names); if (PMIx_Data_compress((uint8_t *) raw, strlen(raw) + 1, (uint8_t **) &bo.bytes, &sz)) { /* mark that this was compressed */ compressed = true; bo.size = sz; free(raw); } else { /* mark that this was not compressed */ compressed = false; bo.bytes = (char *) raw; bo.size = strlen(raw) + 1; } /* indicate compression */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &compressed, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); free(vpids); return rc; } /* add the object */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); free(vpids); return rc; } free(bo.bytes); /* construct the string of aliases for compression */ raw = PMIx_Argv_join(aliases, ';'); PMIx_Argv_free(aliases); if (PMIx_Data_compress((uint8_t *) raw, strlen(raw) + 1, (uint8_t **) &bo.bytes, &sz)) { /* mark that this was compressed */ compressed = true; bo.size = sz; free(raw); } else { /* mark that this was not compressed */ compressed = false; bo.bytes = (char *) raw; bo.size = strlen(raw) + 1; } /* indicate compression */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &compressed, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); free(vpids); return rc; } /* add the object */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); free(vpids); return rc; } free(bo.bytes); /* compress the vpids */ if (PMIx_Data_compress((uint8_t *) vpids, nbytes, (uint8_t **) &bo.bytes, &sz)) { /* mark that this was compressed */ compressed = true; bo.size = sz; free(vpids); } else { /* mark that this was not compressed */ compressed = false; bo.bytes = (char *) vpids; bo.size = ndaemons * sizeof(pmix_rank_t); } /* indicate compression */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &compressed, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); return rc; } /* add the object */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, buffer, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); free(bo.bytes); return rc; } free(bo.bytes); return rc; } int prte_util_decode_nidmap(pmix_data_buffer_t *buf) { uint8_t u8; pmix_rank_t *vpid = NULL; int cnt, n; bool compressed; size_t sz; pmix_byte_object_t pbo; char *raw = NULL, **names = NULL, **aliases = NULL; prte_node_t *nd; prte_job_t *daemons; prte_proc_t *proc; prte_topology_t *t = NULL; pmix_status_t rc; /* unpack the flag indicating if HNP is in allocation */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &u8, &cnt, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } if (1 == u8) { prte_hnp_is_allocated = true; } else { prte_hnp_is_allocated = false; } /* unpack the flag indicating if we are in managed allocation */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &u8, &cnt, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } if (1 == u8) { prte_managed_allocation = true; } else { prte_managed_allocation = false; } /* unpack compression flag for node names */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &compressed, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* unpack the nodename object */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &pbo, &cnt, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* if compressed, decompress */ if (compressed) { if (!PMIx_Data_decompress((uint8_t *) pbo.bytes, pbo.size, (uint8_t **) &raw, &sz)) { PRTE_ERROR_LOG(PRTE_ERROR); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); rc = PRTE_ERROR; goto cleanup; } } else { raw = (char *) pbo.bytes; pbo.bytes = NULL; // protect the data pbo.size = 0; } PMIX_BYTE_OBJECT_DESTRUCT(&pbo); names = PMIx_Argv_split(raw, ','); free(raw); /* unpack compression flag for node aliases */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &compressed, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* unpack the aliases object */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &pbo, &cnt, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* if compressed, decompress */ if (compressed) { if (!PMIx_Data_decompress((uint8_t *) pbo.bytes, pbo.size, (uint8_t **) &raw, &sz)) { PRTE_ERROR_LOG(PRTE_ERROR); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); rc = PRTE_ERROR; goto cleanup; } } else { raw = (char *) pbo.bytes; pbo.bytes = NULL; // protect the data pbo.size = 0; } PMIX_BYTE_OBJECT_DESTRUCT(&pbo); aliases = PMIx_Argv_split(raw, ';'); free(raw); /* unpack compression flag for daemon vpids */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &compressed, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* unpack the vpid object */ cnt = 1; rc = PMIx_Data_unpack(PRTE_PROC_MY_NAME, buf, &pbo, &cnt, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto cleanup; } /* if compressed, decompress */ if (compressed) { if (!PMIx_Data_decompress((uint8_t *) pbo.bytes, pbo.size, (uint8_t **) &vpid, &sz)) { PRTE_ERROR_LOG(PRTE_ERROR); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); rc = PRTE_ERROR; goto cleanup; } } else { vpid = (pmix_rank_t *) pbo.bytes; sz = pbo.size; pbo.bytes = NULL; pbo.size = 0; } PMIX_BYTE_OBJECT_DESTRUCT(&pbo); /* if we are the HNP, we don't need any of this stuff */ if (PRTE_PROC_IS_MASTER) { rc = PRTE_SUCCESS; goto cleanup; } /* get the daemon job object */ daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* get our topology */ t = (prte_topology_t *) pmix_pointer_array_get_item(prte_node_topologies, 0); if (NULL == t) { /* should never happen */ PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto cleanup; } /* create the node pool array - this will include * _all_ nodes known to the allocation */ for (n = 0; NULL != names[n]; n++) { /* do we already have this node? */ nd = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, n); if (NULL != nd) { /* check the name */ if (0 != strcmp(nd->name, names[n])) { free(nd->name); nd->name = strdup(names[n]); } if (0 != strcmp(aliases[n], "PRTENONE")) { if (NULL != nd->aliases) { PMIx_Argv_free(nd->aliases); } nd->aliases = PMIx_Argv_split(aliases[n], ','); } continue; } /* add this name to the pool */ nd = PMIX_NEW(prte_node_t); nd->name = strdup(names[n]); nd->index = n; pmix_pointer_array_set_item(prte_node_pool, n, nd); /* add any aliases */ if (0 != strcmp(aliases[n], "PRTENONE")) { nd->aliases = PMIx_Argv_split(aliases[n], ','); } /* set the topology - always default to homogeneous * as that is the most common scenario */ nd->topology = t; /* record the daemon on it */ proc = (prte_proc_t *) pmix_pointer_array_get_item(daemons->procs, vpid[n]); if (NULL == proc) { proc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&proc->name, PRTE_PROC_MY_NAME->nspace, vpid[n]); proc->state = PRTE_PROC_STATE_RUNNING; PRTE_FLAG_SET(proc, PRTE_PROC_FLAG_ALIVE); daemons->num_procs++; pmix_pointer_array_set_item(daemons->procs, proc->name.rank, proc); } PMIX_RETAIN(nd); proc->node = nd; PMIX_RETAIN(proc); nd->daemon = proc; } /* update num procs */ if (prte_process_info.num_daemons != daemons->num_procs) { prte_process_info.num_daemons = daemons->num_procs; /* update the routing tree */ prte_rml_compute_routing_tree(); } cleanup: if (NULL != vpid) { free(vpid); } if (NULL != names) { PMIx_Argv_free(names); } return rc; } prrte-4.1.1~rc2/src/util/attr.h0000664000175000017500000007202415203542132016533 0ustar alastairalastair/* * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ATTRS_H #define PRTE_ATTRS_H #include "prte_config.h" #include "types.h" /*** FLAG FOR SETTING ATTRIBUTES - INDICATES IF THE *** ATTRIBUTE IS TO BE SHARED WITH REMOTE PROCS OR NOT */ #define PRTE_ATTR_LOCAL true // for local use only #define PRTE_ATTR_GLOBAL false // include when sending this object /* define the mininum value of the PRTE keys just in * case someone someday puts a layer underneath us */ #define PRTE_ATTR_KEY_BASE 0 /*** ATTRIBUTE FLAGS - never sent anywwhere ***/ typedef uint8_t prte_app_context_flags_t; #define PRTE_APP_FLAG_USED_ON_NODE 0x01 // is being used on the local node #define PRTE_APP_FLAG_TOOL 0x02 // this app describes daemons to be co-launched // with the application procs in the other apps // and does not count against allocation #define PRTE_APP_FLAG_COMPUTED 0x04 // num procs for this app were computed and not // given by the user /* APP_CONTEXT ATTRIBUTE KEYS */ #define PRTE_APP_HOSTFILE 1 // string - hostfile #define PRTE_APP_ADD_HOSTFILE 2 // string - hostfile to be added #define PRTE_APP_DASH_HOST 3 // string - hosts specified with -host option #define PRTE_APP_ADD_HOST 4 // string - hosts to be added #define PRTE_APP_USER_CWD 5 // bool - user specified cwd #define PRTE_APP_SSNDIR_CWD 6 // bool - use session dir as cwd #define PRTE_APP_PRELOAD_BIN 7 // bool - move binaries to remote nodes prior to exec #define PRTE_APP_PRELOAD_FILES 8 // string - files to be moved to remote nodes prior to exec #define PRTE_APP_SSTORE_LOAD 9 // string #define PRTE_APP_RECOV_DEF 10 // bool - whether or not a recovery policy was defined #define PRTE_APP_MAX_RESTARTS 11 // int32 - max number of times a process can be restarted #define PRTE_APP_MIN_NODES 12 // int64 - min number of nodes required #define PRTE_APP_MANDATORY 13 // bool - flag if nodes requested in -host are "mandatory" vs "optional" #define PRTE_APP_MAX_PPN 14 // uint32 - maximum number of procs/node for this app #define PRTE_APP_PMIX_PREFIX 15 // string - PMIX_PREFIX value for application procs #define PRTE_APP_NO_CACHEDIR 16 // bool - flag that a cache dir is not to be specified for a Singularity container #define PRTE_APP_SET_ENVAR 17 // prte_envar_t - set the given envar to the specified value #define PRTE_APP_UNSET_ENVAR 18 // string - name of envar to unset, if present #define PRTE_APP_PREPEND_ENVAR 19 // prte_envar_t - prepend the specified value to the given envar #define PRTE_APP_APPEND_ENVAR 20 // prte_envar_t - append the specified value to the given envar #define PRTE_APP_ADD_ENVAR 21 // prte_envar_t - add envar, do not override pre-existing one #define PRTE_APP_PSET_NAME 23 // string - user-assigned name for the process // set containing the given process #define PRTE_APP_PES_PER_PROC 24 // uint16_t - number of cpus to be assigned to each process #define PRTE_APP_PPR 25 // uint16_t - number of procs to place on the resource specified // in the job ppr string #define PRTE_APP_MAX_KEY 100 /*** NODE FLAGS - never sent anywhere ***/ typedef uint8_t prte_node_flags_t; #define PRTE_NODE_FLAG_DAEMON_LAUNCHED 0x01 // whether or not the daemon on this node has been launched #define PRTE_NODE_FLAG_LOC_VERIFIED 0x02 // whether or not the location has been verified - used for // environments where the daemon's final destination is uncertain #define PRTE_NODE_FLAG_OVERSUBSCRIBED 0x04 // whether or not this node is oversubscribed #define PRTE_NODE_FLAG_MAPPED 0x08 // whether we have been added to the current map #define PRTE_NODE_FLAG_SLOTS_GIVEN 0x10 // the number of slots was specified - used only in non-managed environments #define PRTE_NODE_NON_USABLE 0x20 // the node is hosting a tool and is NOT to be used for jobs /*** NODE ATTRIBUTE KEYS - never sent anywhere ***/ #define PRTE_NODE_START_KEY PRTE_APP_MAX_KEY #define PRTE_NODE_USERNAME (PRTE_NODE_START_KEY + 1) #define PRTE_NODE_LAUNCH_ID (PRTE_NODE_START_KEY + 2) // int32 - Launch id needed by some systems to launch a proc on this node #define PRTE_NODE_HOSTID (PRTE_NODE_START_KEY + 3) // pmix_rank_t - if this "node" is a coprocessor being hosted on a different node, then // we need to know the id of our "host" to help any procs on us to determine locality #define PRTE_NODE_SERIAL_NUMBER (PRTE_NODE_START_KEY + 5) // string - serial number: used if node is a coprocessor #define PRTE_NODE_PORT (PRTE_NODE_START_KEY + 6) // int32 - Alternate port to be passed to plm #define PRTE_NODE_ADD_SLOTS (PRTE_NODE_START_KEY + 7) // bool - slots are being added to existing node #define PRTE_NODE_MAX_KEY (PRTE_NODE_START_KEY + 100) /*** JOB FLAGS - included in prte_job_t transmissions ***/ typedef uint16_t prte_job_flags_t; #define PRTE_JOB_FLAGS_T PRTE_UINT16 #define PRTE_JOB_FLAG_UPDATED 0x0001 // job has been updated and needs to be included in the pidmap message #define PRTE_JOB_FLAG_RESTARTED 0x0004 // some procs in this job are being restarted #define PRTE_JOB_FLAG_ABORTED 0x0008 // did this job abort? #define PRTE_JOB_FLAG_FORWARD_OUTPUT 0x0020 // forward output from the apps #define PRTE_JOB_FLAG_DO_NOT_MONITOR 0x0040 // do not monitor apps for termination #define PRTE_JOB_FLAG_FORWARD_COMM 0x0080 // #define PRTE_JOB_FLAG_RESTART 0x0200 // #define PRTE_JOB_FLAG_PROCS_MIGRATING 0x0400 // some procs in job are migrating from one node to another #define PRTE_JOB_FLAG_OVERSUBSCRIBED 0x0800 // at least one node in the job is oversubscribed #define PRTE_JOB_FLAG_TOOL 0x1000 // job is a tool and doesn't count against allocations #define PRTE_JOB_FLAG_LAUNCHER 0x2000 // job is also a launcher #define PRTE_JOB_FLAG_ERR_REPORTED 0x4000 // error report for job has been output /*** JOB ATTRIBUTE KEYS ***/ #define PRTE_JOB_START_KEY PRTE_NODE_MAX_KEY #define PRTE_JOB_LAUNCH_MSG_SENT (PRTE_JOB_START_KEY + 1) // timeval - time launch message was sent #define PRTE_JOB_LAUNCH_MSG_RECVD (PRTE_JOB_START_KEY + 2) // timeval - time launch message was recvd #define PRTE_JOB_MAX_LAUNCH_MSG_RECVD (PRTE_JOB_START_KEY + 3) // timeval - max time for launch msg to be received #define PRTE_JOB_CKPT_STATE (PRTE_JOB_START_KEY + 5) // size_t - ckpt state #define PRTE_JOB_SNAPSHOT_REF (PRTE_JOB_START_KEY + 6) // string - snapshot reference #define PRTE_JOB_SNAPSHOT_LOC (PRTE_JOB_START_KEY + 7) // string - snapshot location #define PRTE_JOB_SNAPC_INIT_BAR (PRTE_JOB_START_KEY + 8) // prte_grpcomm_coll_id_t - collective id #define PRTE_JOB_SNAPC_FINI_BAR (PRTE_JOB_START_KEY + 9) // prte_grpcomm_coll_id_t - collective id #define PRTE_JOB_NUM_NONZERO_EXIT (PRTE_JOB_START_KEY + 10) // int32 - number of procs with non-zero exit codes #define PRTE_SPAWN_TIMEOUT_EVENT (PRTE_JOB_START_KEY + 11) // prte_ptr (prte_timer_t*) - timer event for failure detect/response // if fails to launch #define PRTE_JOB_ABORTED_PROC (PRTE_JOB_START_KEY + 12) // prte_ptr (prte_proc_t*) - proc that caused abort to happen #define PRTE_JOB_MAPPER (PRTE_JOB_START_KEY + 13) // bool - job consists of MapReduce mappers #define PRTE_JOB_REDUCER (PRTE_JOB_START_KEY + 14) // bool - job consists of MapReduce reducers #define PRTE_JOB_COMBINER (PRTE_JOB_START_KEY + 15) // bool - job consists of MapReduce combiners #define PRTE_JOB_INDEX_ARGV (PRTE_JOB_START_KEY + 16) // bool - automatically index argvs #define PRTE_JOB_NO_VM (PRTE_JOB_START_KEY + 17) // bool - do not use VM launch #define PRTE_JOB_SPIN_FOR_DEBUG (PRTE_JOB_START_KEY + 18) // bool - the prted's are to spin while waiting for debugger #define PRTE_JOB_CONTINUOUS (PRTE_JOB_START_KEY + 19) // bool - job consists of continuously operating apps #define PRTE_JOB_RECOVER_DEFINED (PRTE_JOB_START_KEY + 20) // bool - recovery policy has been defined #define PRTE_JOB_NON_PRTE_JOB (PRTE_JOB_START_KEY + 22) // bool - non-prte job #define PRTE_JOB_STDOUT_TARGET (PRTE_JOB_START_KEY + 23) // pmix_nspace_t - job that is to receive the stdout (on its // stdin) from this one #define PRTE_JOB_POWER (PRTE_JOB_START_KEY + 24) // string - power setting for nodes in job #define PRTE_JOB_MAX_FREQ (PRTE_JOB_START_KEY + 25) // string - max freq setting for nodes in job #define PRTE_JOB_MIN_FREQ (PRTE_JOB_START_KEY + 26) // string - min freq setting for nodes in job #define PRTE_JOB_GOVERNOR (PRTE_JOB_START_KEY + 27) // string - governor used for nodes in job #define PRTE_JOB_FAIL_NOTIFIED (PRTE_JOB_START_KEY + 28) // bool - abnormal term of proc within job has been reported #define PRTE_JOB_TERM_NOTIFIED (PRTE_JOB_START_KEY + 29) // bool - normal term of job has been reported #define PRTE_JOB_PEER_MODX_ID (PRTE_JOB_START_KEY + 30) // prte_grpcomm_coll_id_t - collective id #define PRTE_JOB_INIT_BAR_ID (PRTE_JOB_START_KEY + 31) // prte_grpcomm_coll_id_t - collective id #define PRTE_JOB_FINI_BAR_ID (PRTE_JOB_START_KEY + 32) // prte_grpcomm_coll_id_t - collective id #define PRTE_JOB_FWDIO_TO_TOOL (PRTE_JOB_START_KEY + 33) // Forward IO for this job to the tool requesting its spawn #define PRTE_JOB_LAUNCHED_DAEMONS (PRTE_JOB_START_KEY + 35) // bool - Job caused new daemons to be spawned #define PRTE_JOB_REPORT_BINDINGS (PRTE_JOB_START_KEY + 36) // bool - Report process bindings #define PRTE_JOB_CPUSET (PRTE_JOB_START_KEY + 37) // string - "soft" cgroup envelope for the job #define PRTE_JOB_NOTIFICATIONS (PRTE_JOB_START_KEY + 38) // string - comma-separated list of desired notifications+methods #define PRTE_JOB_ROOM_NUM (PRTE_JOB_START_KEY + 39) // int - number of remote request's hotel room #define PRTE_JOB_LAUNCH_PROXY (PRTE_JOB_START_KEY + 40) // pmix_proc_t - name of spawn requestor #define PRTE_JOB_NSPACE_REGISTERED (PRTE_JOB_START_KEY + 41) // bool - job has been registered with embedded PMIx server #define PRTE_JOB_FIXED_DVM (PRTE_JOB_START_KEY + 42) // bool - do not change the size of the DVM for this job #define PRTE_JOB_DVM_JOB (PRTE_JOB_START_KEY + 43) // bool - job is using a DVM #define PRTE_JOB_CANCELLED (PRTE_JOB_START_KEY + 44) // bool - job was cancelled #define PRTE_JOB_OUTPUT_TO_FILE (PRTE_JOB_START_KEY + 45) // string - path to use as basename of files to which // stdout/err is to be directed #define PRTE_JOB_MERGE_STDERR_STDOUT (PRTE_JOB_START_KEY + 46) // bool - merge stderr into stdout stream #define PRTE_JOB_TAG_OUTPUT (PRTE_JOB_START_KEY + 47) // bool - tag stdout/stderr #define PRTE_JOB_TIMESTAMP_OUTPUT (PRTE_JOB_START_KEY + 48) // bool - timestamp stdout/stderr #define PRTE_JOB_MULTI_DAEMON_SIM (PRTE_JOB_START_KEY + 49) // bool - multiple daemons/node to simulate large cluster #define PRTE_JOB_NOTIFY_COMPLETION (PRTE_JOB_START_KEY + 50) // bool - notify parent proc when spawned job terminates #define PRTE_JOB_TRANSPORT_KEY (PRTE_JOB_START_KEY + 51) // string - transport keys assigned to this job #define PRTE_JOB_INFO_CACHE (PRTE_JOB_START_KEY + 52) // pmix_list_t - list of prte_value_t to be included in job_info #define PRTE_JOB_SILENT_TERMINATION (PRTE_JOB_START_KEY + 54) // bool - do not generate an event notification when job // normally terminates #define PRTE_JOB_SET_ENVAR (PRTE_JOB_START_KEY + 55) // prte_envar_t - set the given envar to the specified value #define PRTE_JOB_UNSET_ENVAR (PRTE_JOB_START_KEY + 56) // string - name of envar to unset, if present #define PRTE_JOB_PREPEND_ENVAR (PRTE_JOB_START_KEY + 57) // prte_envar_t - prepend the specified value to the given envar #define PRTE_JOB_APPEND_ENVAR (PRTE_JOB_START_KEY + 58) // prte_envar_t - append the specified value to the given envar #define PRTE_JOB_ADD_ENVAR (PRTE_JOB_START_KEY + 59) // prte_envar_t - add envar, do not override pre-existing one #define PRTE_JOB_APP_SETUP_DATA (PRTE_JOB_START_KEY + 60) // pmix_byte_object_t - blob containing app setup data #define PRTE_JOB_OUTPUT_TO_DIRECTORY (PRTE_JOB_START_KEY + 61) // string - path of directory to which stdout/err is to be directed #define PRTE_JOB_STOP_ON_EXEC (PRTE_JOB_START_KEY + 62) // bool - stop on first instruction for debugger attach #define PRTE_JOB_SPAWN_NOTIFIED (PRTE_JOB_START_KEY + 63) // bool - process requesting a spawn operation has been notified of result #define PRTE_JOB_DISPLAY_MAP (PRTE_JOB_START_KEY + 64) // bool - display job map #define PRTE_JOB_DISPLAY_DEVEL_MAP (PRTE_JOB_START_KEY + 65) // bool - display devel level job map #define PRTE_JOB_DISPLAY_TOPO (PRTE_JOB_START_KEY + 66) // bool - display topology with job map // 67 was removed option diffable map #define PRTE_JOB_DISPLAY_ALLOC (PRTE_JOB_START_KEY + 68) // bool - display allocation #define PRTE_JOB_DO_NOT_LAUNCH (PRTE_JOB_START_KEY + 69) // bool - do not launch job #define PRTE_JOB_XML_OUTPUT (PRTE_JOB_START_KEY + 70) // bool - print in xml format #define PRTE_JOB_TIMEOUT (PRTE_JOB_START_KEY + 71) // int32 - number of seconds job can run before terminating it as timed out #define PRTE_JOB_STACKTRACES (PRTE_JOB_START_KEY + 72) // bool - include process stack traces in timeout report #define PRTE_JOB_REPORT_STATE (PRTE_JOB_START_KEY + 73) // bool - include process state in timeout report #define PRTE_JOB_TIMEOUT_EVENT (PRTE_JOB_START_KEY + 74) // prte_ptr (prte_timer_t*) - timer event for job timeout #define PRTE_JOB_TRACE_TIMEOUT_EVENT (PRTE_JOB_START_KEY + 75) // prte_ptr (prte_timer_t*) - timer event for stacktrace collection #define PRTE_JOB_INHERIT (PRTE_JOB_START_KEY + 76) // bool - job inherits parent's mapping/ranking/binding policies #define PRTE_JOB_PES_PER_PROC (PRTE_JOB_START_KEY + 77) // uint16_t - number of cpus to be assigned to each process #define PRTE_JOB_DIST_DEVICE (PRTE_JOB_START_KEY + 78) // char* - device to use for dist mapping #define PRTE_JOB_HWT_CPUS (PRTE_JOB_START_KEY + 79) // bool - job requests hwthread cpus #define PRTE_JOB_CORE_CPUS (PRTE_JOB_START_KEY + 80) // bool - job requests core cpus #define PRTE_JOB_PPR (PRTE_JOB_START_KEY + 81) // char* - string specifying the procs-per-resource pattern #define PRTE_JOB_NOINHERIT (PRTE_JOB_START_KEY + 82) // bool do NOT inherit parent's mapping/ranking/binding policies #define PRTE_JOB_FILE (PRTE_JOB_START_KEY + 83) // char* - file to use for sequential or rankfile mapping #define PRTE_JOB_DO_NOT_RESOLVE (PRTE_JOB_START_KEY + 84) // bool - do not resolve nodes #define PRTE_JOB_DEBUG_TARGET (PRTE_JOB_START_KEY + 85) // pmix_proc_t - application proc to co-locate daemons with #define PRTE_JOB_DEBUG_DAEMONS_PER_NODE (PRTE_JOB_START_KEY + 86) // uint16_t - Number of debug daemons per node #define PRTE_JOB_DEBUG_DAEMONS_PER_PROC (PRTE_JOB_START_KEY + 87) // uint16_t - Number of debug daemons per application proc #define PRTE_JOB_STOP_IN_INIT (PRTE_JOB_START_KEY + 88) // bool - stop in PMIx_Init #define PRTE_JOB_STOP_IN_APP (PRTE_JOB_START_KEY + 89) // bool - stop at app-determined location #define PRTE_JOB_ENVARS_HARVESTED (PRTE_JOB_START_KEY + 90) // envars have already been harvested #define PRTE_JOB_OUTPUT_NOCOPY (PRTE_JOB_START_KEY + 91) // bool - do not copy output to stdout/err #define PRTE_JOB_RANK_OUTPUT (PRTE_JOB_START_KEY + 92) // bool - tag stdout/stderr with rank #define PRTE_SPAWN_TIMEOUT (PRTE_JOB_START_KEY + 93) // int32 - number of seconds to spawn before terminating it as timed out #define PRTE_JOB_RAW_OUTPUT (PRTE_JOB_START_KEY + 94) // bool - do not buffer output #define PRTE_JOB_EXEC_AGENT (PRTE_JOB_START_KEY + 95) // char* - string specifying the cmd to use when exec'ing the local proc #define PRTE_JOB_NOAGG_HELP (PRTE_JOB_START_KEY + 96) // bool - do not aggregate show_help messages #define PRTE_JOB_COLOCATE_PROCS (PRTE_JOB_START_KEY + 97) // pmix_data_array_t - colocate this job's procs with the given ones #define PRTE_JOB_COLOCATE_NPERPROC (PRTE_JOB_START_KEY + 98) // uint16_t - number of procs to colocate at each proc #define PRTE_JOB_COLOCATE_NPERNODE (PRTE_JOB_START_KEY + 99) // uint16_t - number of procs to colocate on node of each proc #define PRTE_JOB_TAG_OUTPUT_DETAILED (PRTE_JOB_START_KEY + 100) // bool - include [hostname:pid] in output stream tag #define PRTE_JOB_TAG_OUTPUT_FULLNAME (PRTE_JOB_START_KEY + 101) // bool - use full namespace in output stream tag #define PRTE_JOB_ERROR_NONZERO_EXIT (PRTE_JOB_START_KEY + 102) // bool - mark it as an error if a proc exits with non-zero status #define PRTE_JOB_CONTROLS (PRTE_JOB_START_KEY + 103) // char* - Directives controlling job behavior #define PRTE_JOB_SHOW_PROGRESS (PRTE_JOB_START_KEY + 104) // bool - show launch progress of this job #define PRTE_JOB_RECOVERABLE (PRTE_JOB_START_KEY + 105) // bool - job processes can be recovered, do not terminate upon // process failure #define PRTE_JOB_NOTIFY_ERRORS (PRTE_JOB_START_KEY + 106) // bool - provide PMIx events on errors #define PRTE_JOB_AUTORESTART (PRTE_JOB_START_KEY + 107) // bool - automatically restart failed processes #define PRTE_JOB_OUTPUT_PROCTABLE (PRTE_JOB_START_KEY + 108) // char* - string specifying where the output is to go, with a '-' // indicating stdout, '+' indicating stderr, else path #define PRTE_JOB_DISPLAY_PROCESSORS (PRTE_JOB_START_KEY + 109) // char* - string displaying nodes whose avail CPUs // are to be displayed #define PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT (PRTE_JOB_START_KEY + 110) // bool - display output in machine parsable format #define PRTE_JOB_EXTEND_DVM (PRTE_JOB_START_KEY + 111) // bool - DVM is being extended #define PRTE_JOB_SESSION_ID (PRTE_JOB_START_KEY + 112) // uint32_t - session id of this job #define PRTE_JOB_ALLOC_ID (PRTE_JOB_START_KEY + 113) // char* - string identifier assigned by the host for the session // within which the job is to execute #define PRTE_JOB_REF_ID (PRTE_JOB_START_KEY + 114) // char* - string identifier assigned by the user to an allocation // request - carried along with the session that resulted // from the request #define PRTE_JOB_BINDING_LIMIT (PRTE_JOB_START_KEY + 115) // (uint16_t) - Max number of procs to bind to specified // target type before moving to next target #define PRTE_JOB_CHILD_SEP (PRTE_JOB_START_KEY + 116) // bool - child job is to be considered independent // from its parent, do not terminate if // parent dies first #define PRTE_JOB_GPU_SUPPORT (PRTE_JOB_START_KEY + 117) // bool - enable/disable GPU support in app #define PRTE_JOB_PREFIX (PRTE_JOB_START_KEY + 118) // string - PRTE_PREFIX for daemons #define PRTE_JOB_PMIX_PREFIX (PRTE_JOB_START_KEY + 119) // string - PMIX_PREFIX for daemons #define PRTE_JOB_FWD_ENVIRONMENT (PRTE_JOB_START_KEY + 120) // bool - forward local environment to procs in this job #define PRTE_JOB_REPORT_PHYSICAL_CPUS (PRTE_JOB_START_KEY + 121) // bool - report using physical (vs logical) cpu IDs #define PRTE_JOB_ALLOC_DISPLAYED (PRTE_JOB_START_KEY + 122) // bool - allocation has been displayed #define PRTE_JOB_DO_NOT_SPAWN (PRTE_JOB_START_KEY + 123) // bool - do not spawn app procs #define PRTE_JOB_MAX_KEY (PRTE_JOB_START_KEY + 200) /*** PROC FLAGS - never sent anywhere ***/ typedef uint16_t prte_proc_flags_t; #define PRTE_PROC_FLAG_ALIVE 0x0001 // proc has been launched and has not yet terminated #define PRTE_PROC_FLAG_ABORT 0x0002 // proc called abort #define PRTE_PROC_FLAG_UPDATED 0x0004 // proc has been updated and need to be included in the next pidmap message #define PRTE_PROC_FLAG_LOCAL 0x0008 // indicate that this proc is local #define PRTE_PROC_FLAG_REPORTED 0x0010 // indicate proc has reported in #define PRTE_PROC_FLAG_REG 0x0020 // proc has registered #define PRTE_PROC_FLAG_HAS_DEREG 0x0040 // proc has deregistered #define PRTE_PROC_FLAG_AS_MPI 0x0080 // proc is MPI process #define PRTE_PROC_FLAG_IOF_COMPLETE 0x0100 // IOF has completed #define PRTE_PROC_FLAG_WAITPID 0x0200 // waitpid fired #define PRTE_PROC_FLAG_RECORDED 0x0400 // termination has been recorded #define PRTE_PROC_FLAG_DATA_IN_SM 0x0800 // modex data has been stored in the local shared memory region #define PRTE_PROC_FLAG_DATA_RECVD 0x1000 // modex data for this proc has been received #define PRTE_PROC_FLAG_SM_ACCESS 0x2000 // indicate if process can read modex data from shared memory region #define PRTE_PROC_FLAG_TERM_REPORTED 0x4000 // proc termination has been reported /*** PROCESS ATTRIBUTE KEYS ***/ #define PRTE_PROC_START_KEY PRTE_JOB_MAX_KEY #define PRTE_PROC_NOBARRIER (PRTE_PROC_START_KEY + 1) // bool - indicates proc should not barrier in prte_init #define PRTE_PROC_PRIOR_NODE (PRTE_PROC_START_KEY + 5) // void* - pointer to prte_node_t where this proc last executed #define PRTE_PROC_NRESTARTS (PRTE_PROC_START_KEY + 6) // int32 - number of times this process has been restarted #define PRTE_PROC_RESTART_TIME (PRTE_PROC_START_KEY + 7) // timeval - time of last restart #define PRTE_PROC_FAST_FAILS (PRTE_PROC_START_KEY + 8) // int32 - number of failures in "fast" window #define PRTE_PROC_CKPT_STATE (PRTE_PROC_START_KEY + 9) // size_t - ckpt state #define PRTE_PROC_SNAPSHOT_REF (PRTE_PROC_START_KEY + 10) // string - snapshot reference #define PRTE_PROC_SNAPSHOT_LOC (PRTE_PROC_START_KEY + 11) // string - snapshot location #define PRTE_PROC_NODENAME (PRTE_PROC_START_KEY + 12) // string - node where proc is located, used only by tools #define PRTE_PROC_CGROUP (PRTE_PROC_START_KEY + 13) // string - name of cgroup this proc shall be assigned to #define PRTE_PROC_NBEATS (PRTE_PROC_START_KEY + 14) // int32 - number of heartbeats in current window #define PRTE_PROC_MAX_KEY (PRTE_PROC_START_KEY + 100) /*** RML ATTRIBUTE keys ***/ #define PRTE_RML_START_KEY PRTE_PROC_MAX_KEY #define PRTE_RML_TRANSPORT_TYPE (PRTE_RML_START_KEY + 1) // string - null terminated string containing transport type #define PRTE_RML_PROTOCOL_TYPE (PRTE_RML_START_KEY + 2) // string - protocol type (e.g., as returned by fi_info) #define PRTE_RML_CONDUIT_ID (PRTE_RML_START_KEY + 3) // prte_rml_conduit_t - conduit_id for this transport #define PRTE_RML_INCLUDE_COMP_ATTRIB (PRTE_RML_START_KEY + 4) // string - comma delimited list of RML component names to be considered #define PRTE_RML_EXCLUDE_COMP_ATTRIB (PRTE_RML_START_KEY + 5) // string - comma delimited list of RML component names to be excluded #define PRTE_RML_TRANSPORT_ATTRIB (PRTE_RML_START_KEY + 6) // string - comma delimited list of transport types to be considered // (e.g., "fabric,ethernet") #define PRTE_RML_QUALIFIER_ATTRIB (PRTE_RML_START_KEY + 7) // string - comma delimited list of qualifiers (e.g., routed=direct,bandwidth=xxx) #define PRTE_RML_PROVIDER_ATTRIB (PRTE_RML_START_KEY + 8) // string - comma delimited list of provider names to be considered #define PRTE_RML_PROTOCOL_ATTRIB (PRTE_RML_START_KEY + 9) // string - comma delimited list of protocols to be considered (e.g., tcp,udp) #define PRTE_RML_ROUTED_ATTRIB (PRTE_RML_START_KEY + 10) // string - comma delimited list of routed modules to be considered #define PRTE_RML_MAX_KEY (PRTE_RML_START_KEY + 100) #define PRTE_ATTR_KEY_MAX PRTE_RML_MAX_KEY /*** FLAG OPS ***/ #define PRTE_FLAG_SET(p, f) ((p)->flags |= (f)) #define PRTE_FLAG_UNSET(p, f) ((p)->flags &= ~(f)) #define PRTE_FLAG_TEST(p, f) ((p)->flags & (f)) PRTE_EXPORT const char *prte_attr_key_to_str(prte_attribute_key_t key); /* Retrieve the named attribute from a list */ PRTE_EXPORT bool prte_get_attribute(pmix_list_t *attributes, prte_attribute_key_t key, void **data, pmix_data_type_t type); /* Set the named attribute in a list, overwriting any prior entry */ PRTE_EXPORT int prte_set_attribute(pmix_list_t *attributes, prte_attribute_key_t key, bool local, void *data, pmix_data_type_t type); /* Remove the named attribute from a list */ PRTE_EXPORT void prte_remove_attribute(pmix_list_t *attributes, prte_attribute_key_t key); PRTE_EXPORT prte_attribute_t *prte_fetch_attribute(pmix_list_t *attributes, prte_attribute_t *prev, prte_attribute_key_t key); PRTE_EXPORT int prte_prepend_attribute(pmix_list_t *attributes, prte_attribute_key_t key, bool local, void *data, pmix_data_type_t type); PRTE_EXPORT int prte_attr_load(prte_attribute_t *kv, void *data, pmix_data_type_t type); PRTE_EXPORT int prte_attr_unload(prte_attribute_t *kv, void **data, pmix_data_type_t type); PRTE_EXPORT char *prte_attr_print_list(pmix_list_t *attributes); /* * Register a handler for converting attr keys to strings * * Handlers will be invoked by prte_attr_key_to_str to return the appropriate value. */ typedef char *(*prte_attr2str_fn_t)(prte_attribute_key_t key); PRTE_EXPORT int prte_attr_register(const char *project, prte_attribute_key_t key_base, prte_attribute_key_t key_max, prte_attr2str_fn_t converter); /** FOR DIAGNOSTIC PURPOSES **/ #define PRTE_SHOW_ATTRS(a) \ do { \ char *_output = prte_attr_print_list((a)); \ fprintf(stderr, "[%s:%s:%d]\n%s\n", __FILE__, __func__, __LINE__, _output); \ free(_output); \ } while (0) #endif // forward declarations struct prte_proc_t; struct prte_node_t; struct prte_app_context_t; struct prte_job_t; PRTE_EXPORT char* prte_print_proc_flags(struct prte_proc_t *p); PRTE_EXPORT char* prte_print_node_flags(struct prte_node_t *p); PRTE_EXPORT char* prte_print_app_flags(struct prte_app_context_t *p); PRTE_EXPORT char* prte_print_job_flags(struct prte_job_t *p); prrte-4.1.1~rc2/src/util/session_dir.h0000664000175000017500000000402515203542132020076 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * Find and/or create PRTE session directory. * */ #ifndef PRTE_SESSION_DIR_H_HAS_BEEN_INCLUDED #define PRTE_SESSION_DIR_H_HAS_BEEN_INCLUDED #include "prte_config.h" #include "types.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS /** @param proc Pointer to a process name for which the session * dir name is desired. Passing: * * PRTE_NAME_INVALID - top-level session directory * will be created. * * PRTE_NAME_WILDCARD - job-level session directory * will be created * * Valid procID - proc-level session directory will * be created * *@retval PRTE_SUCCESS The directory was found and/or created with * the proper permissions. * @retval PRTE_ERROR The directory cannot be found or created */ PRTE_EXPORT int prte_session_dir(pmix_proc_t *proc); /** The session_dir_finalize functions perform a cleanup of the * relevant session directory tree. */ PRTE_EXPORT void prte_job_session_dir_finalize(prte_job_t *jdata); END_C_DECLS #endif /* PRTE_SESSION_DIR_H_HAS_BEEN_INCLUDED */ prrte-4.1.1~rc2/src/util/proc_info.c0000664000175000017500000002110115203542132017520 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #include #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" #include "src/util/attr.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "src/util/proc_info.h" /* provide a connection to a reqd variable */ extern bool prte_keep_fqdn_hostnames; PRTE_EXPORT prte_process_info_t prte_process_info = { .myproc = PMIX_PROC_STATIC_INIT, .my_hnp = PMIX_PROC_STATIC_INIT, .my_hnp_uri = NULL, .my_parent = PMIX_PROC_STATIC_INIT, .hnp_pid = 0, .num_daemons = 1, .num_nodes = 1, .nodename = NULL, .aliases = NULL, .euid = 0, .egid = 0, .pid = 0, .proc_type = PRTE_PROC_TYPE_NONE, .my_uri = NULL, .my_port = 0, .tmpdir_base = NULL, .sessdir_prefix = NULL, .top_session_dir = NULL, .cpuset = NULL, .shared_fs = false }; static bool init = false; static char *prte_strip_prefix; void prte_setup_hostname(void) { char *ptr; char hostname[PRTE_MAXHOSTNAMELEN]; char **prefixes; bool match; int i, idx; /* whether or not to keep FQDN hostnames */ prte_keep_fqdn_hostnames = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "keep_fqdn_hostnames", "Whether or not to keep FQDN hostnames [default: no]", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_keep_fqdn_hostnames); /* get the nodename */ gethostname(hostname, sizeof(hostname)); prte_strip_prefix = NULL; (void) pmix_mca_base_var_register( "prte", "prte", NULL, "strip_prefix", "Prefix(es) to match when deciding whether to strip leading characters and zeroes from " "node names returned by daemons", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_strip_prefix); /* we have to strip node names here, if user directs, to ensure that * the names exchanged in the modex match the names found locally */ if (NULL != prte_strip_prefix && !pmix_net_isaddr(hostname)) { prefixes = PMIx_Argv_split(prte_strip_prefix, ','); match = false; for (i = 0; NULL != prefixes[i]; i++) { if (0 == strncmp(hostname, prefixes[i], strlen(prefixes[i]))) { /* remove the prefix and leading zeroes */ idx = strlen(prefixes[i]); while (idx < (int) strlen(hostname) && (hostname[idx] <= '0' || '9' < hostname[idx])) { idx++; } if ((int) strlen(hostname) <= idx) { /* there were no non-zero numbers in the name */ prte_process_info.nodename = strdup(&hostname[strlen(prefixes[i])]); } else { prte_process_info.nodename = strdup(&hostname[idx]); } /* add this to our list of aliases */ PMIx_Argv_append_unique_nosize(&prte_process_info.aliases, prte_process_info.nodename); match = true; break; } } /* if we didn't find a match, then just use the hostname as-is */ if (!match) { prte_process_info.nodename = strdup(hostname); } PMIx_Argv_free(prefixes); } else { prte_process_info.nodename = strdup(hostname); } // if we are not keeping FQDN, then strip it off if not an IP address if (!prte_keep_fqdn_hostnames && !pmix_net_isaddr(prte_process_info.nodename)) { ptr = strchr(prte_process_info.nodename, '.'); if (NULL != ptr) { /* add the fqdn name as an alias */ PMIx_Argv_append_unique_nosize(&prte_process_info.aliases, prte_process_info.nodename); /* retain the non-fqdn name as the node's name */ *ptr = '\0'; } } // add the localhost names PMIx_Argv_append_unique_nosize(&prte_process_info.aliases, "localhost"); PMIx_Argv_append_unique_nosize(&prte_process_info.aliases, "127.0.0.1"); } bool prte_check_host_is_local(const char *name) { int i; if (0 == strcmp(name, prte_process_info.nodename)) { return true; } for (i = 0; NULL != prte_process_info.aliases[i]; i++) { if (0 == strcmp(name, prte_process_info.aliases[i])) { return true; } } /* if it wasn't one of those and we are allowed * to resolve addresses, then try that too */ if (!prte_do_not_resolve) { if (pmix_ifislocal(name)) { /* add to our aliases */ PMIx_Argv_append_nosize(&prte_process_info.aliases, name); return true; } } return false; } int prte_proc_info(void) { char *ptr; if (init) { return PRTE_SUCCESS; } init = true; prte_process_info.my_hnp_uri = NULL; pmix_mca_base_var_register("prte", "prte", NULL, "hnp_uri", "HNP contact info", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_process_info.my_hnp_uri); if (NULL != prte_process_info.my_hnp_uri) { ptr = prte_process_info.my_hnp_uri; /* the uri value passed to us will have quote marks around it to protect * the value if passed on the command line. We must remove those * to have a correct uri string */ if ('"' == ptr[0]) { /* if the first char is a quote, then so will the last one be */ ptr[strlen(ptr) - 1] = '\0'; memmove(ptr, ptr + 1, strlen(ptr)); } } /* get the real uid */ prte_process_info.uid = getuid(); /* get the effective uid */ prte_process_info.euid = geteuid(); /* get the real gid */ prte_process_info.gid = getgid(); /* get the effective gid */ prte_process_info.egid = getegid(); /* get the process id */ prte_process_info.pid = getpid(); /* get the number of nodes in the job */ prte_process_info.num_nodes = 1; (void) pmix_mca_base_var_register("prte", "prte", NULL, "num_nodes", "Number of nodes in the job", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_process_info.num_nodes); return PRTE_SUCCESS; } int prte_proc_info_finalize(void) { if (!init) { return PRTE_SUCCESS; } if (NULL != prte_process_info.tmpdir_base) { free(prte_process_info.tmpdir_base); prte_process_info.tmpdir_base = NULL; } if (NULL != prte_process_info.sessdir_prefix) { free(prte_process_info.sessdir_prefix); prte_process_info.sessdir_prefix = NULL; } if (NULL != prte_process_info.top_session_dir) { free(prte_process_info.top_session_dir); prte_process_info.top_session_dir = NULL; } if (NULL != prte_process_info.nodename) { free(prte_process_info.nodename); prte_process_info.nodename = NULL; } if (NULL != prte_process_info.cpuset) { free(prte_process_info.cpuset); prte_process_info.cpuset = NULL; } prte_process_info.proc_type = PRTE_PROC_TYPE_NONE; PMIx_Argv_free(prte_process_info.aliases); init = false; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/util/attr.c0000664000175000017500000010627615203542132016535 0ustar alastairalastair/* * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/attr.h" #define MAX_CONVERTERS 5 #define MAX_CONVERTER_PROJECT_LEN 10 typedef struct { int init; char project[MAX_CONVERTER_PROJECT_LEN]; prte_attribute_key_t key_base; prte_attribute_key_t key_max; prte_attr2str_fn_t converter; } prte_attr_converter_t; /* all default to NULL */ static prte_attr_converter_t converters[MAX_CONVERTERS]; bool prte_get_attribute(pmix_list_t *attributes, prte_attribute_key_t key, void **data, pmix_data_type_t type) { prte_attribute_t *kv; int rc; PMIX_LIST_FOREACH(kv, attributes, prte_attribute_t) { if (key == kv->key) { if (kv->data.type != type) { PRTE_ERROR_LOG(PRTE_ERR_TYPE_MISMATCH); pmix_output(0, "KV %s TYPE %s", PMIx_Data_type_string(kv->data.type), PMIx_Data_type_string(type)); return false; } if (NULL != data) { if (PRTE_SUCCESS != (rc = prte_attr_unload(kv, data, type))) { PRTE_ERROR_LOG(rc); } } return true; } } /* not found */ return false; } int prte_set_attribute(pmix_list_t *attributes, prte_attribute_key_t key, bool local, void *data, pmix_data_type_t type) { prte_attribute_t *kv; bool *bl, bltrue = true; int rc; PMIX_LIST_FOREACH(kv, attributes, prte_attribute_t) { if (key == kv->key) { if (kv->data.type != type) { return PRTE_ERR_TYPE_MISMATCH; } if (PMIX_BOOL == type) { if (NULL == data) { bl = &bltrue; } else { bl = (bool*)data; } if (false == *bl) { pmix_list_remove_item(attributes, &kv->super); PMIX_RELEASE(kv); return PRTE_SUCCESS; } } if (PRTE_SUCCESS != (rc = prte_attr_load(kv, data, type))) { PRTE_ERROR_LOG(rc); } return rc; } } /* not found - add it */ kv = PMIX_NEW(prte_attribute_t); kv->key = key; kv->local = local; if (PRTE_SUCCESS != (rc = prte_attr_load(kv, data, type))) { PMIX_RELEASE(kv); return rc; } pmix_list_append(attributes, &kv->super); return PRTE_SUCCESS; } prte_attribute_t *prte_fetch_attribute(pmix_list_t *attributes, prte_attribute_t *prev, prte_attribute_key_t key) { prte_attribute_t *kv, *end, *next; /* if prev is NULL, then find the first attr on the list * that matches the key */ if (NULL == prev) { PMIX_LIST_FOREACH(kv, attributes, prte_attribute_t) { if (key == kv->key) { return kv; } } /* if we get, then the key isn't on the list */ return NULL; } /* if we are at the end of the list, then nothing to do */ end = (prte_attribute_t *) pmix_list_get_end(attributes); if (prev == end || end == (prte_attribute_t *) pmix_list_get_next(&prev->super) || NULL == pmix_list_get_next(&prev->super)) { return NULL; } /* starting with the next item on the list, search * for the next attr with the matching key */ next = (prte_attribute_t *) pmix_list_get_next(&prev->super); while (NULL != next) { if (next->key == key) { return next; } next = (prte_attribute_t *) pmix_list_get_next(&next->super); } /* if we get here, then no matching key was found */ return NULL; } int prte_prepend_attribute(pmix_list_t *attributes, prte_attribute_key_t key, bool local, void *data, pmix_data_type_t type) { prte_attribute_t *kv; int rc; kv = PMIX_NEW(prte_attribute_t); kv->key = key; kv->local = local; if (PRTE_SUCCESS != (rc = prte_attr_load(kv, data, type))) { PMIX_RELEASE(kv); return rc; } pmix_list_prepend(attributes, &kv->super); return PRTE_SUCCESS; } void prte_remove_attribute(pmix_list_t *attributes, prte_attribute_key_t key) { prte_attribute_t *kv; PMIX_LIST_FOREACH(kv, attributes, prte_attribute_t) { if (key == kv->key) { pmix_list_remove_item(attributes, &kv->super); PMIX_RELEASE(kv); return; } } } int prte_attr_register(const char *project, prte_attribute_key_t key_base, prte_attribute_key_t key_max, prte_attr2str_fn_t converter) { int i; for (i = 0; i < MAX_CONVERTERS; ++i) { if (0 == converters[i].init) { converters[i].init = 1; pmix_string_copy(converters[i].project, project, MAX_CONVERTER_PROJECT_LEN); converters[i].project[MAX_CONVERTER_PROJECT_LEN - 1] = '\0'; converters[i].key_base = key_base; converters[i].key_max = key_max; converters[i].converter = converter; return PRTE_SUCCESS; } } return PRTE_ERR_OUT_OF_RESOURCE; } char *prte_attr_print_list(pmix_list_t *attributes) { char *out1, **cache = NULL; prte_attribute_t *attr; PMIX_LIST_FOREACH(attr, attributes, prte_attribute_t) { PMIx_Argv_append_nosize(&cache, prte_attr_key_to_str(attr->key)); } if (NULL != cache) { out1 = PMIx_Argv_join(cache, '\n'); PMIx_Argv_free(cache); } else { out1 = NULL; } return out1; } static char unknownkey[180] = {0}; const char *prte_attr_key_to_str(prte_attribute_key_t key) { int i; if (PRTE_ATTR_KEY_BASE < key && key < PRTE_ATTR_KEY_MAX) { /* belongs to PRTE, so we handle it */ switch (key) { case PRTE_APP_HOSTFILE: return "APP-HOSTFILE"; case PRTE_APP_ADD_HOSTFILE: return "APP-ADD-HOSTFILE"; case PRTE_APP_DASH_HOST: return "APP-DASH-HOST"; case PRTE_APP_ADD_HOST: return "APP-ADD-HOST"; case PRTE_APP_USER_CWD: return "APP-USER-CWD"; case PRTE_APP_SSNDIR_CWD: return "APP-USE-SESSION-DIR-AS-CWD"; case PRTE_APP_PRELOAD_BIN: return "APP-PRELOAD-BIN"; case PRTE_APP_PRELOAD_FILES: return "APP-PRELOAD-FILES"; case PRTE_APP_SSTORE_LOAD: return "APP-SSTORE-LOAD"; case PRTE_APP_RECOV_DEF: return "APP-RECOVERY-DEFINED"; case PRTE_APP_MAX_RESTARTS: return "APP-MAX-RESTARTS"; case PRTE_APP_MIN_NODES: return "APP-MIN-NODES"; case PRTE_APP_MANDATORY: return "APP-NODES-MANDATORY"; case PRTE_APP_MAX_PPN: return "APP-MAX-PPN"; case PRTE_APP_PMIX_PREFIX: return "APP-PMIX-PREFIX"; case PRTE_APP_NO_CACHEDIR: return "PRTE_APP_NO_CACHEDIR"; case PRTE_APP_SET_ENVAR: return "PRTE_APP_SET_ENVAR"; case PRTE_APP_UNSET_ENVAR: return "PRTE_APP_UNSET_ENVAR"; case PRTE_APP_PREPEND_ENVAR: return "PRTE_APP_PREPEND_ENVAR"; case PRTE_APP_APPEND_ENVAR: return "PRTE_APP_APPEND_ENVAR"; case PRTE_APP_ADD_ENVAR: return "PRTE_APP_ADD_ENVAR"; case PRTE_APP_PSET_NAME: return "PRTE_APP_PSET_NAME"; case PRTE_APP_PES_PER_PROC: return "PRTE_APP_PES_PER_PROC"; case PRTE_APP_PPR: return "PRTE_APP_PPR"; case PRTE_NODE_USERNAME: return "NODE-USERNAME"; case PRTE_NODE_PORT: return "NODE-PORT"; case PRTE_NODE_LAUNCH_ID: return "NODE-LAUNCHID"; case PRTE_NODE_HOSTID: return "NODE-HOSTID"; case PRTE_NODE_SERIAL_NUMBER: return "NODE-SERIAL-NUM"; case PRTE_NODE_ADD_SLOTS: return "NODE-ADD-SLOTS"; case PRTE_JOB_LAUNCH_MSG_SENT: return "JOB-LAUNCH-MSG-SENT"; case PRTE_JOB_LAUNCH_MSG_RECVD: return "JOB-LAUNCH-MSG-RECVD"; case PRTE_JOB_MAX_LAUNCH_MSG_RECVD: return "JOB-MAX-LAUNCH-MSG-RECVD"; case PRTE_JOB_CKPT_STATE: return "JOB-CKPT-STATE"; case PRTE_JOB_SNAPSHOT_REF: return "JOB-SNAPSHOT-REF"; case PRTE_JOB_SNAPSHOT_LOC: return "JOB-SNAPSHOT-LOC"; case PRTE_JOB_SNAPC_INIT_BAR: return "JOB-SNAPC-INIT-BARRIER-ID"; case PRTE_JOB_SNAPC_FINI_BAR: return "JOB-SNAPC-FINI-BARRIER-ID"; case PRTE_JOB_NUM_NONZERO_EXIT: return "JOB-NUM-NONZERO-EXIT"; case PRTE_SPAWN_TIMEOUT_EVENT: return "SPAWN-TIMEOUT-EVENT"; case PRTE_JOB_ABORTED_PROC: return "JOB-ABORTED-PROC"; case PRTE_JOB_MAPPER: return "JOB-MAPPER"; case PRTE_JOB_REDUCER: return "JOB-REDUCER"; case PRTE_JOB_COMBINER: return "JOB-COMBINER"; case PRTE_JOB_INDEX_ARGV: return "JOB-INDEX-ARGV"; case PRTE_JOB_NO_VM: return "JOB-NO-VM"; case PRTE_JOB_SPIN_FOR_DEBUG: return "JOB-SPIN-FOR-DEBUG"; case PRTE_JOB_CONTINUOUS: return "JOB-CONTINUOUS"; case PRTE_JOB_RECOVER_DEFINED: return "JOB-RECOVERY-DEFINED"; case PRTE_JOB_NON_PRTE_JOB: return "JOB-NON-PRTE-JOB"; case PRTE_JOB_STDOUT_TARGET: return "JOB-STDOUT-TARGET"; case PRTE_JOB_POWER: return "JOB-POWER"; case PRTE_JOB_MAX_FREQ: return "JOB-MAX_FREQ"; case PRTE_JOB_MIN_FREQ: return "JOB-MIN_FREQ"; case PRTE_JOB_GOVERNOR: return "JOB-FREQ-GOVERNOR"; case PRTE_JOB_FAIL_NOTIFIED: return "JOB-FAIL-NOTIFIED"; case PRTE_JOB_TERM_NOTIFIED: return "JOB-TERM-NOTIFIED"; case PRTE_JOB_PEER_MODX_ID: return "JOB-PEER-MODX-ID"; case PRTE_JOB_INIT_BAR_ID: return "JOB-INIT-BAR-ID"; case PRTE_JOB_FINI_BAR_ID: return "JOB-FINI-BAR-ID"; case PRTE_JOB_FWDIO_TO_TOOL: return "JOB-FWD-IO-TO-TOOL"; case PRTE_JOB_LAUNCHED_DAEMONS: return "JOB-LAUNCHED-DAEMONS"; case PRTE_JOB_REPORT_BINDINGS: return "JOB-REPORT-BINDINGS"; case PRTE_JOB_CPUSET: return "JOB-CPUSET"; case PRTE_JOB_NOTIFICATIONS: return "JOB-NOTIFICATIONS"; case PRTE_JOB_ROOM_NUM: return "JOB-ROOM-NUM"; case PRTE_JOB_LAUNCH_PROXY: return "JOB-LAUNCH-PROXY"; case PRTE_JOB_NSPACE_REGISTERED: return "JOB-NSPACE-REGISTERED"; case PRTE_JOB_FIXED_DVM: return "PRTE-JOB-FIXED-DVM"; case PRTE_JOB_DVM_JOB: return "PRTE-JOB-DVM-JOB"; case PRTE_JOB_CANCELLED: return "PRTE-JOB-CANCELLED"; case PRTE_JOB_OUTPUT_TO_FILE: return "PRTE-JOB-OUTPUT-TO-FILE"; case PRTE_JOB_MERGE_STDERR_STDOUT: return "PRTE-JOB-MERGE-STDERR-STDOUT"; case PRTE_JOB_TAG_OUTPUT: return "PRTE-JOB-TAG-OUTPUT"; case PRTE_JOB_RANK_OUTPUT: return "PRTE-JOB-RANK-OUTPUT"; case PRTE_JOB_TIMESTAMP_OUTPUT: return "PRTE-JOB-TIMESTAMP-OUTPUT"; case PRTE_JOB_MULTI_DAEMON_SIM: return "PRTE_JOB_MULTI_DAEMON_SIM"; case PRTE_JOB_NOTIFY_COMPLETION: return "PRTE_JOB_NOTIFY_COMPLETION"; case PRTE_JOB_TRANSPORT_KEY: return "PRTE_JOB_TRANSPORT_KEY"; case PRTE_JOB_INFO_CACHE: return "PRTE_JOB_INFO_CACHE"; case PRTE_JOB_SILENT_TERMINATION: return "PRTE_JOB_SILENT_TERMINATION"; case PRTE_JOB_SET_ENVAR: return "PRTE_JOB_SET_ENVAR"; case PRTE_JOB_UNSET_ENVAR: return "PRTE_JOB_UNSET_ENVAR"; case PRTE_JOB_PREPEND_ENVAR: return "PRTE_JOB_PREPEND_ENVAR"; case PRTE_JOB_APPEND_ENVAR: return "PRTE_JOB_APPEND_ENVAR"; case PRTE_JOB_ADD_ENVAR: return "PRTE_APP_ADD_ENVAR"; case PRTE_JOB_APP_SETUP_DATA: return "PRTE_JOB_APP_SETUP_DATA"; case PRTE_JOB_OUTPUT_TO_DIRECTORY: return "PRTE_JOB_OUTPUT_TO_DIRECTORY"; case PRTE_JOB_STOP_ON_EXEC: return "JOB_STOP_ON_EXEC"; case PRTE_JOB_SPAWN_NOTIFIED: return "JOB_SPAWN_NOTIFIED"; case PRTE_JOB_DISPLAY_MAP: return "DISPLAY_JOB_MAP"; case PRTE_JOB_DISPLAY_DEVEL_MAP: return "DISPLAY_DEVEL_JOB_MAP"; case PRTE_JOB_DISPLAY_TOPO: return "DISPLAY_TOPOLOGY"; case PRTE_JOB_DISPLAY_ALLOC: return "DISPLAY_ALLOCATION"; case PRTE_JOB_DO_NOT_LAUNCH: return "DO_NOT_LAUNCH"; case PRTE_JOB_XML_OUTPUT: return "XML_OUTPUT"; case PRTE_JOB_TIMEOUT: return "JOB_TIMEOUT"; case PRTE_JOB_STACKTRACES: return "JOB_STACKTRACES"; case PRTE_JOB_REPORT_STATE: return "JOB_REPORT_STATE"; case PRTE_JOB_TIMEOUT_EVENT: return "JOB_TIMEOUT_EVENT"; case PRTE_JOB_TRACE_TIMEOUT_EVENT: return "JOB_TRACE_TIMEOUT_EVENT"; case PRTE_JOB_INHERIT: return "JOB_INHERIT"; case PRTE_JOB_PES_PER_PROC: return "JOB_PES_PER_PROC"; case PRTE_JOB_DIST_DEVICE: return "JOB_DIST_DEVICE"; case PRTE_JOB_HWT_CPUS: return "JOB_HWT_CPUS"; case PRTE_JOB_CORE_CPUS: return "JOB_CORE_CPUS"; case PRTE_JOB_PPR: return "JOB_PPR"; case PRTE_JOB_NOINHERIT: return "JOB_NOINHERIT"; case PRTE_JOB_FILE: return "JOB-FILE"; case PRTE_JOB_DO_NOT_RESOLVE: return "DO-NOT-RESOLVE"; case PRTE_JOB_DEBUG_TARGET: return "DEBUG-TARGET"; case PRTE_JOB_DEBUG_DAEMONS_PER_NODE: return "DEBUG-DAEMONS-PER-NODE"; case PRTE_JOB_DEBUG_DAEMONS_PER_PROC: return "DEBUG-DAEMONS-PER-PROC"; case PRTE_JOB_STOP_IN_INIT: return "STOP-IN-INIT"; case PRTE_JOB_STOP_IN_APP: return "STOP-IN-APP"; case PRTE_JOB_ENVARS_HARVESTED: return "ENVARS-HARVESTED"; case PRTE_JOB_OUTPUT_NOCOPY: return "DO-NOT-COPY-OUTPUT"; case PRTE_SPAWN_TIMEOUT: return "SPAWN-TIMEOUT"; case PRTE_JOB_RAW_OUTPUT: return "DO-NOT-BUFFER-OUTPUT"; case PRTE_JOB_EXEC_AGENT: return "EXEC-AGENT"; case PRTE_JOB_NOAGG_HELP: return "DO-NOT-AGGREGATE-HELP"; case PRTE_JOB_COLOCATE_PROCS: return "COLOCATE PROCS"; case PRTE_JOB_COLOCATE_NPERPROC: return "NUM PROCS TO COLOCATE PER PROC"; case PRTE_JOB_COLOCATE_NPERNODE: return "NUM PROCS TO COLOCATE PER NODE"; case PRTE_JOB_TAG_OUTPUT_DETAILED: return "DETAILED OUTPUT TAG"; case PRTE_JOB_TAG_OUTPUT_FULLNAME: return "FULL NSPACE IN OUTPUT TAG"; case PRTE_JOB_ERROR_NONZERO_EXIT: return "ERROR IF NONZERO EXIT"; case PRTE_JOB_CONTROLS: return "JOB CONTROLS"; case PRTE_JOB_SHOW_PROGRESS: return "SHOW LAUNCH PROGRESS"; case PRTE_JOB_RECOVERABLE: return "JOB IS RECOVERABLE"; case PRTE_JOB_NOTIFY_ERRORS: return "NOTIFY ERRORS"; case PRTE_JOB_AUTORESTART: return "AUTORESTART"; case PRTE_JOB_OUTPUT_PROCTABLE: return "OUTPUT PROCTABLE"; case PRTE_JOB_DISPLAY_PROCESSORS: return "DISPLAY PROCESSORS"; case PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT: return "DISPLAY PARSEABLE OUTPUT"; case PRTE_JOB_EXTEND_DVM: return "EXTEND DVM"; case PRTE_JOB_SESSION_ID: return "SESSION ID"; case PRTE_JOB_ALLOC_ID: return "ALLOC ID"; case PRTE_JOB_REF_ID: return "ALLOC REF ID"; case PRTE_JOB_BINDING_LIMIT: return "JOB BINDING LIMIT"; case PRTE_JOB_CHILD_SEP: return "CHILD SEP"; case PRTE_JOB_GPU_SUPPORT: return "GPU SUPPORT"; case PRTE_JOB_PREFIX: return "PRTE-JOB-PREFIX"; case PRTE_JOB_PMIX_PREFIX: return "PRTE-JOB-PMIX-PREFIX"; case PRTE_JOB_FWD_ENVIRONMENT: return "FWD ENVIRONMENT"; case PRTE_JOB_REPORT_PHYSICAL_CPUS: return "REPORT PHYSICAL CPUS"; case PRTE_JOB_ALLOC_DISPLAYED: return "ALLOCATION DISPLAYED"; case PRTE_JOB_DO_NOT_SPAWN: return "DO_NOT_SPAWN"; case PRTE_PROC_NOBARRIER: return "PROC-NOBARRIER"; case PRTE_PROC_PRIOR_NODE: return "PROC-PRIOR-NODE"; case PRTE_PROC_NRESTARTS: return "PROC-NUM-RESTARTS"; case PRTE_PROC_RESTART_TIME: return "PROC-RESTART-TIME"; case PRTE_PROC_FAST_FAILS: return "PROC-FAST-FAILS"; case PRTE_PROC_CKPT_STATE: return "PROC-CKPT-STATE"; case PRTE_PROC_SNAPSHOT_REF: return "PROC-SNAPHOT-REF"; case PRTE_PROC_SNAPSHOT_LOC: return "PROC-SNAPSHOT-LOC"; case PRTE_PROC_NODENAME: return "PROC-NODENAME"; case PRTE_PROC_CGROUP: return "PROC-CGROUP"; case PRTE_PROC_NBEATS: return "PROC-NBEATS"; case PRTE_RML_TRANSPORT_TYPE: return "RML-TRANSPORT-TYPE"; case PRTE_RML_PROTOCOL_TYPE: return "RML-PROTOCOL-TYPE"; case PRTE_RML_CONDUIT_ID: return "RML-CONDUIT-ID"; case PRTE_RML_INCLUDE_COMP_ATTRIB: return "RML-INCLUDE"; case PRTE_RML_EXCLUDE_COMP_ATTRIB: return "RML-EXCLUDE"; case PRTE_RML_TRANSPORT_ATTRIB: return "RML-TRANSPORT"; case PRTE_RML_QUALIFIER_ATTRIB: return "RML-QUALIFIER"; case PRTE_RML_PROVIDER_ATTRIB: return "RML-DESIRED-PROVIDERS"; case PRTE_RML_PROTOCOL_ATTRIB: return "RML-DESIRED-PROTOCOLS"; case PRTE_RML_ROUTED_ATTRIB: return "RML-DESIRED-ROUTED-MODULES"; default: pmix_snprintf(unknownkey, 180, "UNKNOWN-KEY: %d", key); return unknownkey; } } /* see if one of the converters can handle it */ for (i = 0; i < MAX_CONVERTERS; ++i) { if (0 != converters[i].init) { if (converters[i].key_base < key && key < converters[i].key_max) { return converters[i].converter(key); } } } /* get here if nobody know what to do */ pmix_snprintf(unknownkey, 180, "UNKNOWN-KEY: %d", key); return unknownkey; } int prte_attr_load(prte_attribute_t *kv, void *data, pmix_data_type_t type) { pmix_byte_object_t *boptr; struct timeval *tv; pmix_envar_t *envar; pmix_status_t rc; kv->data.type = type; if (NULL == data) { /* if the type is BOOL, then the user wanted to * use the presence of the attribute to indicate * "true" - so let's mark it that way just in * case a subsequent test looks for the value */ if (PMIX_BOOL == type) { kv->data.data.flag = true; } else { /* otherwise, check to see if this type has storage * that is already allocated, and free it if so */ if (PMIX_STRING == type && NULL != kv->data.data.string) { free(kv->data.data.string); } else if (PMIX_BYTE_OBJECT == type && NULL != kv->data.data.bo.bytes) { free(kv->data.data.bo.bytes); } else if (PMIX_PROC_NSPACE == type && NULL != kv->data.data.proc) { PMIX_PROC_FREE(kv->data.data.proc, 1); } else if (PMIX_PROC == type && NULL != kv->data.data.proc) { PMIX_PROC_FREE(kv->data.data.proc, 1); } else if (PMIX_ENVAR == type) { if (NULL != kv->data.data.envar.envar) { free(kv->data.data.envar.envar); } if (NULL != kv->data.data.envar.value) { free(kv->data.data.envar.value); } } /* just set the fields to zero */ memset(&kv->data.data, 0, sizeof(kv->data.data)); } return PRTE_SUCCESS; } switch (type) { case PMIX_BOOL: kv->data.data.flag = *(bool *) (data); break; case PMIX_BYTE: kv->data.data.byte = *(uint8_t *) (data); break; case PMIX_STRING: if (NULL != kv->data.data.string) { free(kv->data.data.string); } kv->data.data.string = strdup((const char *) data); break; case PMIX_SIZE: kv->data.data.size = *(size_t *) (data); break; case PMIX_PID: kv->data.data.pid = *(pid_t *) (data); break; case PMIX_INT: kv->data.data.integer = *(int *) (data); break; case PMIX_INT8: kv->data.data.int8 = *(int8_t *) (data); break; case PMIX_INT16: kv->data.data.int16 = *(int16_t *) (data); break; case PMIX_INT32: kv->data.data.int32 = *(int32_t *) (data); break; case PMIX_INT64: kv->data.data.int64 = *(int64_t *) (data); break; case PMIX_UINT: kv->data.data.uint = *(unsigned int *) (data); break; case PMIX_UINT8: kv->data.data.uint8 = *(uint8_t *) (data); break; case PMIX_UINT16: kv->data.data.uint16 = *(uint16_t *) (data); break; case PMIX_UINT32: kv->data.data.uint32 = *(uint32_t *) data; break; case PMIX_UINT64: kv->data.data.uint64 = *(uint64_t *) (data); break; case PMIX_BYTE_OBJECT: if (NULL != kv->data.data.bo.bytes) { free(kv->data.data.bo.bytes); } boptr = (pmix_byte_object_t *) data; if (NULL != boptr && NULL != boptr->bytes && 0 < boptr->size) { kv->data.data.bo.bytes = (char *) malloc(boptr->size); memcpy(kv->data.data.bo.bytes, boptr->bytes, boptr->size); kv->data.data.bo.size = boptr->size; } else { kv->data.data.bo.bytes = NULL; kv->data.data.bo.size = 0; } break; case PMIX_FLOAT: kv->data.data.fval = *(float *) (data); break; case PMIX_TIMEVAL: tv = (struct timeval *) data; kv->data.data.tv.tv_sec = tv->tv_sec; kv->data.data.tv.tv_usec = tv->tv_usec; break; case PMIX_POINTER: kv->data.data.ptr = data; break; case PMIX_PROC_RANK: kv->data.data.rank = *(pmix_rank_t *) data; break; case PMIX_PROC_NSPACE: if (NULL == kv->data.data.proc) { PMIX_PROC_CREATE(kv->data.data.proc, 1); if (NULL == kv->data.data.proc) { return PRTE_ERR_OUT_OF_RESOURCE; } } PMIX_LOAD_NSPACE(kv->data.data.proc->nspace, (char *) data); break; case PMIX_PROC: if (NULL == kv->data.data.proc) { PMIX_PROC_CREATE(kv->data.data.proc, 1); if (NULL == kv->data.data.proc) { return PRTE_ERR_OUT_OF_RESOURCE; } } PMIX_XFER_PROCID(kv->data.data.proc, (pmix_proc_t *) data); break; case PMIX_ENVAR: envar = (pmix_envar_t *) data; if (NULL != envar) { if (NULL != kv->data.data.envar.envar) { free(kv->data.data.envar.envar); } if (NULL != kv->data.data.envar.value) { free(kv->data.data.envar.value); } if (NULL != envar->envar) { kv->data.data.envar.envar = strdup(envar->envar); } if (NULL != envar->value) { kv->data.data.envar.value = strdup(envar->value); } kv->data.data.envar.separator = envar->separator; } break; case PMIX_DATA_ARRAY: rc = PMIx_Data_copy((void**)&kv->data.data.darray, data, PMIX_DATA_ARRAY); return rc; break; default: PRTE_ERROR_LOG(PRTE_ERR_NOT_SUPPORTED); return PRTE_ERR_NOT_SUPPORTED; } return PRTE_SUCCESS; } int prte_attr_unload(prte_attribute_t *kv, void **data, pmix_data_type_t type) { pmix_byte_object_t *boptr; pmix_envar_t *envar; pmix_data_array_t *darray; pmix_status_t rc; pmix_data_type_t pointers[] = { PMIX_STRING, PMIX_BYTE_OBJECT, PMIX_POINTER, PMIX_PROC_NSPACE, PMIX_PROC, PMIX_ENVAR, PMIX_DATA_ARRAY, PMIX_UNDEF}; int n; bool found = false; if (type != kv->data.type) { return PRTE_ERR_TYPE_MISMATCH; } if (NULL == data) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } /* if they didn't give us a storage address * and the data type isn't one where we can * create storage, then this is an error */ for (n = 0; PMIX_UNDEF != pointers[n]; n++) { if (type == pointers[n]) { found = true; break; } } if (!found && NULL == *data) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } switch (type) { case PMIX_BOOL: memcpy(*data, &kv->data.data.flag, sizeof(bool)); break; case PMIX_BYTE: memcpy(*data, &kv->data.data.byte, sizeof(uint8_t)); break; case PMIX_STRING: if (NULL != kv->data.data.string) { *data = strdup(kv->data.data.string); } else { *data = NULL; } break; case PMIX_SIZE: memcpy(*data, &kv->data.data.size, sizeof(size_t)); break; case PMIX_PID: memcpy(*data, &kv->data.data.pid, sizeof(pid_t)); break; case PMIX_INT: memcpy(*data, &kv->data.data.integer, sizeof(int)); break; case PMIX_INT8: memcpy(*data, &kv->data.data.int8, sizeof(int8_t)); break; case PMIX_INT16: memcpy(*data, &kv->data.data.int16, sizeof(int16_t)); break; case PMIX_INT32: memcpy(*data, &kv->data.data.int32, sizeof(int32_t)); break; case PMIX_INT64: memcpy(*data, &kv->data.data.int64, sizeof(int64_t)); break; case PMIX_UINT: memcpy(*data, &kv->data.data.uint, sizeof(unsigned int)); break; case PMIX_UINT8: memcpy(*data, &kv->data.data.uint8, 1); break; case PMIX_UINT16: memcpy(*data, &kv->data.data.uint16, 2); break; case PMIX_UINT32: memcpy(*data, &kv->data.data.uint32, 4); break; case PMIX_UINT64: memcpy(*data, &kv->data.data.uint64, 8); break; case PMIX_BYTE_OBJECT: boptr = (pmix_byte_object_t *) malloc(sizeof(pmix_byte_object_t)); if (NULL == boptr) { return PRTE_ERR_OUT_OF_RESOURCE; } if (NULL != kv->data.data.bo.bytes && 0 < kv->data.data.bo.size) { boptr->bytes = (char *) malloc(kv->data.data.bo.size); memcpy(boptr->bytes, kv->data.data.bo.bytes, kv->data.data.bo.size); boptr->size = kv->data.data.bo.size; } else { boptr->bytes = NULL; boptr->size = 0; } *data = boptr; break; case PMIX_FLOAT: memcpy(*data, &kv->data.data.fval, sizeof(float)); break; case PMIX_TIMEVAL: memcpy(*data, &kv->data.data.tv, sizeof(struct timeval)); break; case PMIX_POINTER: *data = kv->data.data.ptr; break; case PMIX_PROC_RANK: memcpy(*data, &kv->data.data.rank, sizeof(pmix_rank_t)); break; case PMIX_PROC_NSPACE: PMIX_PROC_CREATE(*data, 1); if (NULL == *data) { return PRTE_ERR_OUT_OF_RESOURCE; } memcpy(*data, kv->data.data.proc->nspace, sizeof(pmix_nspace_t)); break; case PMIX_PROC: PMIX_PROC_CREATE(*data, 1); if (NULL == *data) { return PRTE_ERR_OUT_OF_RESOURCE; } memcpy(*data, kv->data.data.proc, sizeof(pmix_proc_t)); break; case PMIX_ENVAR: PMIX_ENVAR_CREATE(envar, 1); if (NULL == envar) { return PRTE_ERR_OUT_OF_RESOURCE; } if (NULL != kv->data.data.envar.envar) { envar->envar = strdup(kv->data.data.envar.envar); } if (NULL != kv->data.data.envar.value) { envar->value = strdup(kv->data.data.envar.value); } envar->separator = kv->data.data.envar.separator; *data = envar; break; case PMIX_DATA_ARRAY: rc = PMIx_Data_copy((void**)&darray, kv->data.data.darray, PMIX_DATA_ARRAY); if (PMIX_SUCCESS != rc) { *data = NULL; return prte_pmix_convert_status(rc); } *data = darray; break; default: PRTE_ERROR_LOG(PRTE_ERR_NOT_SUPPORTED); return PRTE_ERR_NOT_SUPPORTED; } return PRTE_SUCCESS; } char* prte_print_proc_flags(struct prte_proc_t *ptr) { prte_proc_t *p = (prte_proc_t*)ptr; char **tmp = NULL; char *ans; // start with the proc name PMIx_Argv_append_nosize(&tmp, PRTE_NAME_PRINT(&p->name)); PMIx_Argv_append_nosize(&tmp, ": "); if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_ALIVE)) { PMIx_Argv_append_nosize(&tmp, "ALIVE"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_ABORT)) { PMIx_Argv_append_nosize(&tmp, "ABORT"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_UPDATED)) { PMIx_Argv_append_nosize(&tmp, "UPDATED"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_LOCAL)) { PMIx_Argv_append_nosize(&tmp, "LOCAL"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_REPORTED)) { PMIx_Argv_append_nosize(&tmp, "REPORTED"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_REG)) { PMIx_Argv_append_nosize(&tmp, "REGISTERED"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_HAS_DEREG)) { PMIx_Argv_append_nosize(&tmp, "DEREGISTERED"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_AS_MPI)) { PMIx_Argv_append_nosize(&tmp, "MPI"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_IOF_COMPLETE)) { PMIx_Argv_append_nosize(&tmp, "IOF-COMPLETE"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_WAITPID)) { PMIx_Argv_append_nosize(&tmp, "WAITPID"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_RECORDED)) { PMIx_Argv_append_nosize(&tmp, "RECORDED"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_DATA_IN_SM)) { PMIx_Argv_append_nosize(&tmp, "DATA-IN-SM"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_DATA_RECVD)) { PMIx_Argv_append_nosize(&tmp, "DATA-RECVD"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_SM_ACCESS)) { PMIx_Argv_append_nosize(&tmp, "SM-ACCESS"); } if (PRTE_FLAG_TEST(p, PRTE_PROC_FLAG_TERM_REPORTED)) { PMIx_Argv_append_nosize(&tmp, "TERMINATED"); } ans = PMIx_Argv_join(tmp, '|'); PMIx_Argv_free(tmp); return ans; } char* prte_print_node_flags(struct prte_node_t *ptr) { prte_node_t *p = (prte_node_t*)ptr; char **tmp = NULL; char *ans; // start with the node name PMIx_Argv_append_nosize(&tmp, p->name); PMIx_Argv_append_nosize(&tmp, ": "); if (PRTE_FLAG_TEST(p, PRTE_NODE_FLAG_DAEMON_LAUNCHED)) { PMIx_Argv_append_nosize(&tmp, "DAEMON-LAUNCHED"); } if (PRTE_FLAG_TEST(p, PRTE_NODE_FLAG_LOC_VERIFIED)) { PMIx_Argv_append_nosize(&tmp, "LOCATION"); } if (PRTE_FLAG_TEST(p, PRTE_NODE_FLAG_OVERSUBSCRIBED)) { PMIx_Argv_append_nosize(&tmp, "OVERSUBSCRIBED"); } if (PRTE_FLAG_TEST(p, PRTE_NODE_FLAG_MAPPED)) { PMIx_Argv_append_nosize(&tmp, "MAPPED"); } if (PRTE_FLAG_TEST(p, PRTE_NODE_FLAG_SLOTS_GIVEN)) { PMIx_Argv_append_nosize(&tmp, "SLOTS-GIVEN"); } if (PRTE_FLAG_TEST(p, PRTE_NODE_NON_USABLE)) { PMIx_Argv_append_nosize(&tmp, "NONUSABLE"); } ans = PMIx_Argv_join(tmp, '|'); PMIx_Argv_free(tmp); return ans; } char* prte_print_app_flags(struct prte_app_context_t *ptr) { prte_app_context_t *p = (prte_app_context_t*)ptr; char **tmp = NULL; char *ans; // start with the app command PMIx_Argv_append_nosize(&tmp, p->app); PMIx_Argv_append_nosize(&tmp, ": "); if (PRTE_FLAG_TEST(p, PRTE_APP_FLAG_USED_ON_NODE)) { PMIx_Argv_append_nosize(&tmp, "USED-LOCAL-NODE"); } if (PRTE_FLAG_TEST(p, PRTE_APP_FLAG_TOOL)) { PMIx_Argv_append_nosize(&tmp, "TOOL"); } if (PRTE_FLAG_TEST(p, PRTE_APP_FLAG_COMPUTED)) { PMIx_Argv_append_nosize(&tmp, "NPROCS-COMPUTED"); } ans = PMIx_Argv_join(tmp, '|'); PMIx_Argv_free(tmp); return ans; } char* prte_print_job_flags(struct prte_job_t *ptr) { prte_job_t *p = (prte_job_t*)ptr; char **tmp = NULL; char *ans; // start with the job name PMIx_Argv_append_nosize(&tmp, PRTE_JOBID_PRINT(p->nspace)); PMIx_Argv_append_nosize(&tmp, ": "); if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_UPDATED)) { PMIx_Argv_append_nosize(&tmp, "UPDATED"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_RESTARTED)) { PMIx_Argv_append_nosize(&tmp, "RESTARTED"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_ABORTED)) { PMIx_Argv_append_nosize(&tmp, "ABORTED"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_FORWARD_OUTPUT)) { PMIx_Argv_append_nosize(&tmp, "FORWARD-OUTPUT"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_DO_NOT_MONITOR)) { PMIx_Argv_append_nosize(&tmp, "DO-NOT-MONITOR"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_FORWARD_COMM)) { PMIx_Argv_append_nosize(&tmp, "FWD-COM"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_RESTART)) { PMIx_Argv_append_nosize(&tmp, "RESTART"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_PROCS_MIGRATING)) { PMIx_Argv_append_nosize(&tmp, "MIGRATING"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_OVERSUBSCRIBED)) { PMIx_Argv_append_nosize(&tmp, "OVERSUBSCRIBED"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_TOOL)) { PMIx_Argv_append_nosize(&tmp, "TOOL"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_LAUNCHER)) { PMIx_Argv_append_nosize(&tmp, "LAUNCHER"); } if (PRTE_FLAG_TEST(p, PRTE_JOB_FLAG_ERR_REPORTED)) { PMIx_Argv_append_nosize(&tmp, "ERROR-REPORTED"); } ans = PMIx_Argv_join(tmp, '|'); PMIx_Argv_free(tmp); return ans; } prrte-4.1.1~rc2/src/util/session_dir.c0000664000175000017500000002445715203542132020104 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_PWD_H # include #endif #include #include #include #ifdef HAVE_SYS_PARAM_H # include #endif /* HAVE_SYS_PARAM_H */ #ifdef HAVE_SYS_TYPES_H # include #endif /* HAVE_SYS_TYPES_H */ #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #ifdef HAVE_DIRENT_H # include #endif /* HAVE_DIRENT_H */ #ifdef HAVE_PWD_H # include #endif /* HAVE_PWD_H */ #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_os_path.h" #include "src/util/pmix_output.h" #include "src/util/pmix_path.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_environ.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ras/base/base.h" #include "src/runtime/prte_globals.h" #include "src/runtime/runtime.h" #include "src/util/session_dir.h" /******************************* * Local function Declarations *******************************/ static bool _check_file(const char *root, const char *path); static bool setup_base_complete = false; #define PRTE_PRINTF_FIX_STRING(a) ((NULL == a) ? "(null)" : a) /**************************** * Funcationality ****************************/ /* * Check and create the directory requested */ static int _create_dir(char *directory) { mode_t my_mode = S_IRWXU; /* I'm looking for full rights */ int ret; /* attempt to create it */ ret = pmix_os_dirpath_create(directory, my_mode); if (PMIX_ERR_EXISTS == ret) { // existence is good enough ret = PMIX_SUCCESS; } else if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); } ret = prte_pmix_convert_status(ret); return ret; } static int _setup_top_session_dir(void) { int rc = PRTE_SUCCESS; /* get the effective uid */ pid_t pid = getpid(); /* construct the top_session_dir if we need */ if (NULL == prte_process_info.top_session_dir) { if (NULL == prte_process_info.nodename || NULL == prte_process_info.tmpdir_base) { /* we can't setup top session dir */ rc = PRTE_ERR_BAD_PARAM; goto exit; } if (prte_process_info.shared_fs) { /* if it is a shared fs, then we need to include our nodename * to avoid collisions */ if (0 > pmix_asprintf(&prte_process_info.top_session_dir, "%s/%s.%s.%lu", prte_process_info.tmpdir_base, prte_process_info.sessdir_prefix, prte_process_info.nodename, (unsigned long)pid)) { prte_process_info.top_session_dir = NULL; rc = PRTE_ERR_OUT_OF_RESOURCE; goto exit; } } else { /* if not a shared fs, then we can just use our pid as that will be unique */ if (0 > pmix_asprintf(&prte_process_info.top_session_dir, "%s/%s.%lu", prte_process_info.tmpdir_base, prte_process_info.sessdir_prefix, (unsigned long)pid)) { prte_process_info.top_session_dir = NULL; rc = PRTE_ERR_OUT_OF_RESOURCE; goto exit; } } } rc = _create_dir(prte_process_info.top_session_dir); exit: if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); if (NULL != prte_process_info.top_session_dir) { free(prte_process_info.top_session_dir); prte_process_info.top_session_dir = NULL; } } return rc; } static int _setup_job_session_dir(prte_job_t *jdata) { int rc = PRTE_SUCCESS; if (NULL == jdata->session_dir) { if (0 > pmix_asprintf(&jdata->session_dir, "%s/%s", prte_process_info.top_session_dir, PRTE_LOCAL_JOBID_PRINT(jdata->nspace))) { return PRTE_ERR_OUT_OF_RESOURCE; } rc = _create_dir(jdata->session_dir); } return rc; } static int _setup_proc_session_dir(prte_job_t *jdata, pmix_proc_t *p) { int rc; char *tmp; if (0 > pmix_asprintf(&tmp, "%s/%s", jdata->session_dir, PMIX_RANK_PRINT(p->rank))) { return PRTE_ERR_OUT_OF_RESOURCE; } rc = _create_dir(tmp); free(tmp); return rc; } static int setup_base(void) { int rc; // only do this once if (setup_base_complete) { return PRTE_SUCCESS; } setup_base_complete = true; /* Ensure that system info is set */ prte_proc_info(); /* BEFORE doing anything else, check to see if this prefix is * allowed by the system */ if (NULL != prte_prohibited_session_dirs || NULL != prte_process_info.tmpdir_base) { char **list; int i, len; /* break the string into tokens - it should be * separated by ',' */ list = PMIx_Argv_split(prte_prohibited_session_dirs, ','); len = PMIx_Argv_count(list); /* cycle through the list */ for (i = 0; i < len; i++) { /* check if prefix matches */ if (0 == strncmp(prte_process_info.tmpdir_base, list[i], strlen(list[i]))) { /* this is a prohibited location */ pmix_show_help("help-prte-runtime.txt", "prte:session:dir:prohibited", true, prte_process_info.tmpdir_base, prte_prohibited_session_dirs); PMIx_Argv_free(list); return PRTE_ERR_FATAL; } } PMIx_Argv_free(list); /* done with this */ } rc = _setup_top_session_dir(); return rc; } /* * Construct the session directory and create it if necessary */ int prte_session_dir(pmix_proc_t *proc) { int rc = PRTE_SUCCESS; prte_job_t *jdata; /* * Get the session directory full name */ if (PRTE_SUCCESS != (rc = setup_base())) { if (PRTE_ERR_FATAL == rc) { /* this indicates we should abort quietly */ rc = PRTE_ERR_SILENT; } goto cleanup; } /* setup job and proc session directories */ jdata = prte_get_job_data_object(proc->nspace); if (NULL == jdata) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } if (PRTE_SUCCESS != (rc = _setup_job_session_dir(jdata))) { PRTE_ERROR_LOG(rc); return rc; } if (PMIX_RANK_IS_VALID(proc->rank)) { if (PRTE_SUCCESS != (rc = _setup_proc_session_dir(jdata, proc))) { PRTE_ERROR_LOG(rc); return rc; } } if (prte_debug_flag) { pmix_output(0, "jobdir: %s", PRTE_PRINTF_FIX_STRING(jdata->session_dir)); pmix_output(0, "top: %s", PRTE_PRINTF_FIX_STRING(prte_process_info.top_session_dir)); pmix_output(0, "tmp: %s", PRTE_PRINTF_FIX_STRING(prte_process_info.tmpdir_base)); } cleanup: return rc; } void prte_job_session_dir_finalize(prte_job_t *jdata) { if (prte_process_info.rm_session_dirs) { /* RM will clean them up for us */ return; } /* special case - if a daemon is colocated with mpirun, * then we let mpirun do the rest to avoid a race * condition. this scenario always results in the rank=1 * daemon colocated with mpirun */ if (prte_ras_base.launch_orted_on_hn && PRTE_PROC_IS_DAEMON && 1 == PRTE_PROC_MY_NAME->rank) { return; } /* if this is the DVM job, then we destroy the top-level * session directory, but only if we are finalizing */ if (NULL == jdata || PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, jdata->nspace)) { if (prte_finalizing) { if (NULL != prte_process_info.top_session_dir) { pmix_os_dirpath_destroy(prte_process_info.top_session_dir, true, _check_file); /* if the top session dir is now empty, remove it */ rmdir(prte_process_info.top_session_dir); free(prte_process_info.top_session_dir); prte_process_info.top_session_dir = NULL; } } return; } if (NULL == jdata || NULL == jdata->session_dir) { return; } pmix_os_dirpath_destroy(jdata->session_dir, true, _check_file); /* if the job-level session dir is now empty, remove it */ rmdir(jdata->session_dir); free(jdata->session_dir); jdata->session_dir = NULL; return; } static bool _check_file(const char *root, const char *path) { struct stat st; char *fullpath; /* * Keep: * - non-zero files starting with "output-" */ if (0 == strncmp(path, "output-", strlen("output-"))) { memset(&st, 0, sizeof(struct stat)); fullpath = pmix_os_path(false, root, path, NULL); if (0 != stat(fullpath, &st)) { pmix_output(0, "%s Syscall failure for stat: %s(%d)", PMIX_NAME_PRINT(&pmix_globals.myid), strerror(errno), errno); return true; } free(fullpath); if (0 == st.st_size) { return true; } return false; } return true; } prrte-4.1.1~rc2/src/util/uri.c0000664000175000017500000001055115203542132016350 0ustar alastairalastair/* * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #include #ifdef HAVE_UNISTD_H # include #endif #include "src/util/pmix_output.h" #include "src/util/pmix_path.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_show_help.h" #include "src/util/uri.h" static const char *uri_reserved_path_chars = "!$&'()*+,;=:@ "; char *prte_uri_get_scheme(const char *uri) { char *turi = strdup(uri); char *ptr; if (NULL == (ptr = strchr(turi, ':'))) { pmix_show_help("help-prte-util.txt", "malformed-uri", true, uri); free(turi); return NULL; } *ptr = '\0'; return turi; } char *prte_filename_to_uri(const char *filename, const char *hostname) { char *uri, *fn; size_t i, j, k, n; /* filename must be an absolute path */ if (!pmix_path_is_absolute(filename)) { pmix_show_help("help-prte-util.txt", "relative-path", true, filename); return NULL; } /* if hostname is NULL, then this is a local file, so * the scheme can either be missing or given as "localhost" */ if (NULL == hostname) { pmix_asprintf(&uri, "file://%s", filename); return uri; } /* count the number of characters that require escaping * in the filename */ n = 0; for (j = 0; j < strlen(uri_reserved_path_chars) - 1; j++) { if (NULL != strchr(filename, uri_reserved_path_chars[j])) { n++; } } /* escape them if necessary */ if (0 < n) { fn = (char *) malloc(strlen(filename) + n + 1); i = 0; for (k = 0; k < strlen(filename) - 1; k++) { for (j = 0; j < strlen(uri_reserved_path_chars) - 1; j++) { if (filename[k] == uri_reserved_path_chars[j]) { fn[i] = '\\'; i++; break; } } fn[i] = filename[k]; i++; } fn[i] = '\0'; } else { fn = strdup(filename); } /* construct the uri - the filename was already tested to * ensure it was absolute, so the required separator should * already be present */ pmix_asprintf(&uri, "file://%s%s", hostname, fn); free(fn); return uri; } char *prte_filename_from_uri(const char *uri, char **hostname) { char *turi; char *ptr, *fn, *sp; /* protect the incoming string */ turi = strdup(uri); /* set defaults */ fn = NULL; if (NULL != hostname) { *hostname = NULL; } /* extract the scheme */ if (NULL == (ptr = strchr(turi, ':'))) { pmix_show_help("help-prte-util.txt", "malformed-uri", true, uri); free(turi); return NULL; } *ptr = '\0'; ptr++; /* step over the new NULL */ /* if there are three '/', then there is no * hostname and the file is local */ if (0 == strncmp(ptr, "///", 3)) { /* step to the filename - as it is required * to be an absolute path, leave one slash * in the name */ ptr += 2; fn = strdup(ptr); } else if (0 != strncmp(ptr, "//", 2)) { /* error */ pmix_show_help("help-prte-util.txt", "malformed-uri", true, uri); } else { ptr += 2; /* step to the hostname */ /* find the separator to the filename */ if (NULL == (sp = strchr(ptr, '/'))) { pmix_show_help("help-prte-util.txt", "malformed-uri", true, uri); } else { *sp = '\0'; if (NULL != hostname) { *hostname = strdup(ptr); } /* the filename is required to be an * absolute path, so restore the slash */ *sp = '/'; fn = strdup(sp); } } free(turi); return fn; } prrte-4.1.1~rc2/src/util/stacktrace.h0000664000175000017500000000371515203542132017706 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * @file */ #ifndef PRTE_STACKTRACE_H #define PRTE_STACKTRACE_H #include "prte_config.h" /* * File descriptor to be used by the backtrace framework if prte_backtrace_print * is passed NULL for it's FILE file pointer. */ extern int prte_stacktrace_output_fileno; /** * Output the current stack trace (not including the call to this * function) to the stream indicated. */ PRTE_EXPORT void prte_stackframe_output(int stream); /** * Return the current stack trace (not including the call to this * function) as a string (which must be freed by the caller). */ PRTE_EXPORT char *prte_stackframe_output_string(void); /** * Here we register the prte_show_stackframe function for signals * passed to OpenMPI by the mpi_signal-parameter passed to mpirun * by the user. * * @returnvalue PRTE_SUCCESS * @returnvalue PRTE_ERR_BAD_PARAM if the value in the signal-list * is not a valid signal-number * */ PRTE_EXPORT int prte_util_register_stackhandlers(void); #endif /* PRTE_STACKTRACE_H */ prrte-4.1.1~rc2/src/util/daemon_init.c0000664000175000017500000000601315203542132020035 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Geoffroy Vallee. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2020 Triad National Security, LLC. All rights * reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #include #include #ifdef HAVE_UNISTD_H # include #endif #include #include "constants.h" #include "src/util/daemon_init.h" int prte_daemon_init_callback(char *working_dir, int (*parent_fn)(pid_t)) { #if defined(HAVE_FORK) pid_t pid; int fd; if ((pid = fork()) < 0) { return PRTE_ERROR; } else if (pid != 0) { /* parent goes bye-bye */ int rc = 0; if (NULL != parent_fn) { rc = parent_fn(pid); } exit(rc); } /* child continues */ # if defined(HAVE_SETSID) setsid(); /* become session leader - doing this confuses Cray aprun in some cases */ # endif if (NULL != working_dir) { if (-1 == chdir(working_dir)) { /* change working directory */ return PRTE_ERR_FATAL; } } /* connect input to /dev/null */ fd = open("/dev/null", O_RDONLY); if (0 > fd) { return PRTE_ERR_FATAL; } dup2(fd, STDIN_FILENO); if (fd != STDIN_FILENO) { close(fd); } /* connect outputs to /dev/null */ fd = open("/dev/null", O_RDWR | O_CREAT | O_TRUNC, 0666); if (fd >= 0) { dup2(fd, STDOUT_FILENO); dup2(fd, STDERR_FILENO); /* just to be safe, make sure we aren't trying * to close stdout or stderr! since we dup'd both * of them to the same fd, we can't just close it * since one of the two would still be open and * someone could attempt to use it. */ if (fd != STDOUT_FILENO && fd != STDERR_FILENO) { close(fd); } } else { return PRTE_ERR_FATAL; } return PRTE_SUCCESS; #else /* HAVE_FORK */ return PRTE_ERR_NOT_SUPPORTED; #endif } prrte-4.1.1~rc2/src/util/ethtool.c0000664000175000017500000000430715203542132017231 0ustar alastairalastair/* * Copyright (c) 2016 Karol Mroz. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_SOCKET_H # include #endif #ifdef HAVE_NET_IF_H # include #endif #ifdef HAVE_LINUX_ETHTOOL_H # include #endif #ifdef HAVE_SYS_IOCTL_H # include #endif #ifdef HAVE_LINUX_SOCKIOS_H # include #endif #include "src/runtime/prte_globals.h" #include "src/util/ethtool.h" #include "src/util/pmix_if.h" #include "src/util/pmix_string_copy.h" /* * Obtain an appropriate bandwidth for the interface if_name. On Linux, we * get this via an ioctl(). Elsewhere or in the error case, we return the * speed as 0. */ unsigned int prte_ethtool_get_speed(const char *if_name) { unsigned int speed = 0; #if defined(HAVE_DECL_SIOCETHTOOL) && defined(HAVE_STRUCT_IFREQ) && defined(HAVE_STRUCT_ETHTOOL_CMD) int sockfd; struct ifreq ifr; struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET, }; sockfd = socket(PF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { return 0; } memset(&ifr, 0, sizeof(struct ifreq)); pmix_string_copy(ifr.ifr_name, if_name, PMIX_IF_NAMESIZE); ifr.ifr_data = (char *) &edata; if (ioctl(sockfd, SIOCETHTOOL, &ifr) < 0) { goto out; } # if HAVE_DECL_ETHTOOL_CMD_SPEED speed = ethtool_cmd_speed(&edata); # elif defined(HAVE_STRUCT_ETHTOOL_CMD_SPEED_HI) speed = (edata.speed_hi << 16) | edata.speed; # else speed = edata.speed; # endif if (UINT_MAX == speed) { speed = 0; } out: close(sockfd); #else PRTE_HIDE_UNUSED_PARAMS(if_name); #endif return speed; } prrte-4.1.1~rc2/src/util/numtostr.c0000664000175000017500000000350215203542132017442 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/util/numtostr.h" #include "src/util/pmix_printf.h" #include #include char *prte_ltostr(long num) { /* waste a little bit of space, but always have a big enough buffer */ int buflen = sizeof(long) * 8; char *buf = NULL; int ret = 0; buf = (char *) malloc(sizeof(char) * buflen); if (NULL == buf) return NULL; ret = snprintf(buf, buflen, "%ld", num); if (ret < 0) { free(buf); return NULL; } return buf; } char *prte_dtostr(double num) { /* waste a little bit of space, but always have a big enough buffer */ int buflen = sizeof(long) * 8; char *buf = NULL; int ret = 0; buf = (char *) malloc(sizeof(char) * buflen); if (NULL == buf) return NULL; ret = snprintf(buf, buflen, "%f", num); if (ret < 0) { free(buf); return NULL; } return buf; } prrte-4.1.1~rc2/src/util/error_strings.h0000664000175000017500000000306415203542132020461 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * */ #ifndef _PRTE_ERROR_STRINGS_H_ #define _PRTE_ERROR_STRINGS_H_ #include "prte_config.h" #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS PRTE_EXPORT const char *prte_job_state_to_str(prte_job_state_t state); PRTE_EXPORT const char *prte_app_ctx_state_to_str(prte_app_state_t state); PRTE_EXPORT const char *prte_proc_state_to_str(prte_proc_state_t state); PRTE_EXPORT const char *prte_node_state_to_str(prte_node_state_t state); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/prte-convert-help.py0000775000175000017500000005052415203542132021344 0ustar alastairalastair#!/usr/bin/env python3 # # Copyright (c) 2025 Jeffrey M. Squyres. All rights reserved. # Copyright (c) 2025-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # from __future__ import print_function import os import sys import argparse def parse_cmd_line_options(path, verbose=False): # read the file and parse it for options options = [] with open(path) as file: for line in file: stripped = line.strip() if stripped.startswith("#define"): # find the first space at end of "define" s1 = stripped.find(' ') # first word after "#define " is the name of the option start = 8 end = stripped.find(' ', start+1) if -1 == end: continue name = stripped[start:end] # find the first quote mark start = stripped.find('"', end) if start != -1: # find the ending quote mark end = stripped.find('"', start+1) # everything in-between is the actual option opt = stripped[start+1:end] # track it options.append((name, opt)) return options def find_files(root, verbose=False): # Check for existence of root directory - otherwise, we just # fall through with no error output if not os.path.isdir(root): print("Root directory " + root + " does not exist\nCannot continue") exit(1) # Search for help-*.txt files across the source tree, skipping # some directories (e.g., .git) # Also find and retrieve all source files (.h or .c) # so we can search them for entries help_files = [] source_files = [] tool_help_files = [] tool_source_files = [] # skip infrastructure directories skip_dirs = ['.git', '.libs', '.deps'] # there may also be some help files we want to ignore skip_files = [] for root_dir, dirs, files in os.walk(root): for sd in skip_dirs: if sd in dirs: dirs.remove(sd) for file in files: if file.startswith("help-") and file.endswith(".txt"): skipit = False for sf in skip_files: if sf == file: skipit = True break if not skipit: full_path = os.path.join(root_dir, file) if "schizo/prte" in full_path or "schizo/ompi" in full_path or "tools" in full_path: tool_help_files.append(full_path) if verbose: print("Found tool help: {full_path}".format(full_path=full_path)) else: help_files.append(full_path) if verbose: print("Found help: {full_path}".format(full_path=full_path)) elif file.endswith(".c") or file.endswith(".h"): full_path = os.path.join(root_dir, file) if "schizo/prte" in full_path or "schizo/ompi" in full_path or "tools" in full_path: tool_source_files.append(full_path) else: source_files.append(full_path) if verbose: print("Found source: {full_path}".format(full_path=full_path)) return help_files, source_files, tool_help_files, tool_source_files def parse_help_files(file_paths, data, citations, verbose=False): # Parse INI-style help files, returning a dictionary with filenames as # keys. Don't use the Python configparse module in order to # reduce dependencies (i.e., so that we don't have to pip install # anything to run this script). for file_path in file_paths: sections = {} current_section = None with open(file_path) as file: for line in file: stripped = line.rstrip() if not stripped and current_section is None: continue if stripped.startswith("#include"): # if we aren't in a section, then this is an error if current_section is None: sys.stderr.write("ERROR: INCLUDE is specified in ", file_path, " outside of a section") continue # include the line in the section sections[current_section].append(stripped) # this line includes a file/topic from another file # find the '#' at the end of the "include" keyword start = stripped.find('#', 2) if start != -1: # find the '#'' at the end of the filename end = stripped.find('#', start+1) end2 = stripped.find('#', end+1) if end2 == -1: topic = stripped[end+1:] fil = stripped[start+1:end] citations.append((fil, topic)) else: topic = stripped[end2+1:] fil = stripped[end+1:end2] citations.append((fil, topic)) # if a section contains an include, we treat that # section as having been cited citations.append((os.path.basename(file_path), current_section)) continue if stripped.startswith('#'): current_section = None continue if stripped.startswith('['): # find the end of the section name end = stripped.find(']', 1) if -1 == end: continue current_section = stripped[1:end] sections[current_section] = list() elif current_section is not None: sections[current_section].append(stripped) if file_path in data: sys.stderr.write("ERROR: path ", file_path, " already exists in data dictionary") else: data[file_path] = sections if verbose: p = len(sections) print("Parsed: {file_path} ({p} sections found)".format(file_path=file_path, p=p)) return def parse_src_files(source_files, citations, verbose=False): # search the source code for show_help references for src in source_files: cont_topic = False cont_filename = False with open(src) as file: for line in file: line = line.strip() # skip the obvious comment lines if line.startswith("//") or line.startswith("/*") or line.startswith("* ") or line.startswith("PRTE_EXPORT"): continue if cont_topic: # the topic for the prior filename should be on this line if verbose: print("CONT TOPIC LINE: ", line) start = line.find('"') if start != -1: end = line.find('"', start+1, -1) if end != -1: topic = line[start+1:end] citations.append((filename,topic)) if verbose: print("Found topic: ", filename, topic) cont_topic = False continue elif cont_filename: # the filename should be on this line start = line.find('"') if start != -1: end = line.find('"', start+1, -1) if end != -1: filename = line[start+1:end] # see if the topic is on the same line start = line.find('"', end+1, -1) if start != -1: # find the end of it end = line.find('"', start+1, -1) if end != -1: topic = line[start+1:end] citations.append((filename,topic)) else: # topic must be on next line cont_topic = True if verbose: print("Found filename: ", filename) cont_filename = False continue else: cont_filename = False sys.stderr.write("ERROR: Missing end of filename") continue if "pmix_show_help(" in line or "pmix_show_help_string(" in line or "send_error_show_help" in line: cont_topic = False cont_filename = False # line contains call to show-help - try to extract # filename/topic start = line.find('"') if start != -1: end = line.find('"', start+1, -1) if end != -1: filename = line[start+1:end] # see if the topic is on this line start = line.find('"', end+1, -1) if start != -1: # find the end of it end = line.find('"', start+1, -1) if end != -1: topic = line[start+1:end] citations.append((filename,topic)) else: # topic must be on next line cont_topic = True continue else: # the filename must be on next line cont_filename = True continue return def parse_tool_files(help_files, citations, verbose=False): # cycle through the tool help files for hlp in help_files: topics = [] options = [] inUsage = False # scan the help file to find cmd line options that # are referenced in the help file itself with open(hlp) as file: continuation = False for line in file: line = line.rstrip() if not line: continue if line.startswith('#'): inUsage = False if line.startswith('[') and line.endswith(']'): # topic - save it start = line.find('[') + 1 end = line.find(']') topic = line[start:end] topics.append(topic) if "usage" == topic: inUsage = True continue if not inUsage: continue if not line.startswith('|'): continue # look for end of option entryend = line.find('|', 2) # if this is a continuation line, then just take the remaining option words if continuation: optend = line.find(' ', 2, entryend) # if the last character is a quote, then back up if line[optend-1] == '"': optend = optend - 1 opt = optstart + line[2:optend] options.append(opt) continuation = False continue # scan the line for the dash(es) that indicates a cmd line option opt = line.find('-', 2, entryend) if -1 == opt: continue # check for single-dash option if '-' != line[opt+1]: options.append(line[opt+1]) # adjust the entryend value as there will be a double-dash version entryend = line.find('|', opt+6) # there could be a double-dash version as well opt = line.find("--", 2, entryend) if -1 == opt: continue # move past the double-dashes opt += 2 optend = line.find(' ', opt+1, entryend) if line[optend-1] == '"': optend = optend - 1 # if the last character is a "-", then the option continues on next line if line[optend-1] == '-': optstart = line[opt:optend] continuation = True continue option = line[opt:optend] # skip the help, verbose, and version standard options if option == "help" or option == "version" or option == "verbose": continue # track the option options.append(option) # scan the options for option in options: # scan the topics for a match for topic in topics: if option == topic: if verbose: print("CITED ", os.path.basename(hlp), option) citations.append((os.path.basename(hlp), option)) break def purge(parsed_data, citations): special_topics = ["help", "version", "usage", "placement", "placement-examples", "placement-rankfiles", "placement-deprecated", "placement-diagnostics", "placement-fundamentals", "placement-limits"] result_data = {} errorFound = False for filename in parsed_data: sections = parsed_data[filename] result_sections = {} for section in sections: content_list = sections[section] # check for duplicate entries content = '\n'.join(content_list) # search all other entries for a matching section for (file2, sec) in parsed_data.items(): if file2 == filename: continue for (sec2, cl) in sec.items(): cnt = '\n'.join(cl) if sec == section: if content == cnt: # these are the same sys.stderr.write("DUPLICATE FOUND:\n SECTION: " + section + "\n FILES: " + \ filename + "\n " + file2 + "\n") errorFound = True else: # same topic, different content sys.stderr.write("DUPLICATE SECTION WITH DIFFERENT CONTENT:\n SECTION: " + \ section + "\n FILES: " + filename + "\n " + file2 + "\n") errorFound = True # search code files for usage # protect special values if section in special_topics: result_sections[section] = content_list continue for entry in citations: used = False file2 = entry[0] sec = entry[1] if file2 == os.path.basename(filename) and sec == section: # this is a used entry result_sections[section] = content_list used = True break; if not used: sys.stderr.write("** WARNING: Unused help topic\n") sys.stderr.write(" File: " + filename + "\n") sys.stderr.write(" Section: " + section + "\n") errorFound = True # see if anything is left if 0 < len(result_sections): # don't retain the file if no citations for it are left result_data[filename] = result_sections else: sys.stderr.write("File " + filename + " has no used topics - omitting\n") errorFound = True if errorFound: exit(1) if errorFound: exit(1) return result_data def generate_c_code(parsed_data): # Generate C code with an array of filenames and their # corresponding INI sections. c_code = "// THIS FILE IS GENERATED AUTOMATICALLY! EDITS WILL BE LOST!\n" + \ "// This file generated by " + os.path.basename(sys.argv[0]) + "\n\n" c_code += "#include \"src/include/prte_config.h\"\n" + \ "#include \n" + \ "#include \n\n" + \ "#include \"src/include/pmix_globals.h\"\n\n" ini_arrays = [] file_entries = [] for idx, (fil, sections) in enumerate(sorted(parsed_data.items())): filename = os.path.basename(fil) var_name = filename.replace('-', '_').replace('.', '_') ini_entries = [] for section, content_list in sections.items(): entry = " { .topic = \"" + section + "\",\n .content = (const char *[]){\n" for content in content_list: c_content = content.replace('"','\\"').replace("\n", '\\n"\n"').replace("\\;", ";") entry += " \"" + c_content + "\",\n" entry += " NULL}\n },\n" ini_entries.append(entry) ini_entries.append(" { .topic = NULL, .content = NULL }") ini_array_name = "ini_entries_" + str(idx) ini_arrays.append("static pmix_show_help_entry_t " + ini_array_name + "[] = {\n" + "\n".join(ini_entries) + "\n};\n") file_entries.append(" { \"" + filename + "\", " + ini_array_name + "}") file_entries.append(" { NULL, NULL }") c_code += "\n".join(ini_arrays) + "\n" c_code += "pmix_show_help_file_t prte_show_help_data[] = {\n" + ",\n".join(file_entries) + "\n};\n" return c_code #------------------------------- def main(): parser = argparse.ArgumentParser(description="Generate C code from help text INI files.") parser.add_argument("--root", required=True, help="Root directory to search for help-*.txt files") parser.add_argument("--out", required=True, help="Output C file") parser.add_argument("--verbose", action="store_true", help="Enable verbose output") parser.add_argument("--purge", action="store_true", help="Purge duplicates, update topic pointers as required") parser.add_argument("--dryrun", action="store_true", help="Do not write out resulting ini_array_name") args = parser.parse_args() parsed_data = {} citations = [] cli_options = [] outdata = {} rootdir = os.path.join(args.root, "src") # the options are in root/util/prte_cmd_line.h path = os.path.join(rootdir, "util", "prte_cmd_line.h") if not os.path.exists(path): sys.stderr.write("File " + path + " does not exist\nCannot continue") exit(1) # obtain a list of (option name, string) tuples cli_options = parse_cmd_line_options(path, args.verbose) help_files, source_files, tool_help_files, tool_source_files = find_files(rootdir, args.verbose) parse_help_files(help_files, parsed_data, citations, args.verbose) parse_help_files(tool_help_files, parsed_data, citations, args.verbose) parse_src_files(source_files, citations, args.verbose) parse_src_files(tool_source_files, citations, args.verbose) parse_tool_files(tool_help_files, citations, args.verbose) if args.purge: outdata = purge(parsed_data, citations) else: outdata = parsed_data if not args.dryrun: c_code = generate_c_code(outdata) with open(args.out, "w") as f: f.write(c_code) if args.verbose: print("Generated C code written to ", args.out) if __name__ == "__main__": main() prrte-4.1.1~rc2/src/util/parse_options.c0000664000175000017500000001055715203542132020444 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_parse_options.h" void pmix_util_parse_range_options(char *inp, char ***output) { char **r1 = NULL, **r2 = NULL; int i, vint; int start, end, n; char nstr[32]; char *input, *bang; bool bang_option = false; /* protect against null input */ if (NULL == inp) { return; } /* protect the provided input */ input = strdup(inp); /* check for the special '!' operator */ if (NULL != (bang = strchr(input, '!'))) { bang_option = true; *bang = '\0'; } /* split on commas */ r1 = PMIx_Argv_split(input, ','); /* for each resulting element, check for range */ for (i = 0; i < PMIx_Argv_count(r1); i++) { r2 = PMIx_Argv_split(r1[i], '-'); if (1 < PMIx_Argv_count(r2)) { /* given range - get start and end */ start = strtol(r2[0], NULL, 10); end = strtol(r2[1], NULL, 10); } else { /* check for wildcard - have to do this here because * the -1 would have been caught in the split */ vint = strtol(r1[i], NULL, 10); if (-1 == vint) { PMIx_Argv_free(*output); *output = NULL; PMIx_Argv_append_nosize(output, "-1"); PMIx_Argv_free(r2); goto cleanup; } start = strtol(r2[0], NULL, 10); end = start; } for (n = start; n <= end; n++) { snprintf(nstr, 32, "%d", n); PMIx_Argv_append_nosize(output, nstr); } PMIx_Argv_free(r2); } cleanup: if (bang_option) { PMIx_Argv_append_nosize(output, "BANG"); } free(input); PMIx_Argv_free(r1); } void prte_util_get_ranges(char *inp, char ***startpts, char ***endpts) { char **r1 = NULL, **r2 = NULL; int i; char *input; /* protect against null input */ if (NULL == inp) { return; } /* protect the provided input */ input = strdup(inp); /* split on commas */ r1 = PMIx_Argv_split(input, ','); /* for each resulting element, check for range */ for (i = 0; i < PMIx_Argv_count(r1); i++) { r2 = PMIx_Argv_split(r1[i], '-'); if (2 == PMIx_Argv_count(r2)) { /* given range - get start and end */ PMIx_Argv_append_nosize(startpts, r2[0]); PMIx_Argv_append_nosize(endpts, r2[1]); } else if (1 == PMIx_Argv_count(r2)) { /* only one value provided, so it is both the start * and the end */ PMIx_Argv_append_nosize(startpts, r2[0]); PMIx_Argv_append_nosize(endpts, r2[0]); } else { /* no idea how to parse this */ pmix_output(0, "%s Unknown parse error on string: %s(%s)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), inp, r1[i]); } PMIx_Argv_free(r2); } free(input); PMIx_Argv_free(r1); } prrte-4.1.1~rc2/src/util/crc.c0000664000175000017500000013723315203542132016327 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * Copyright (c) 2021 FUJITSU LIMITED. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #ifdef HAVE_STDIO_H # include #endif /* HAVE_STDIO_H */ #include #ifdef HAVE_STRINGS_H # include #endif /* HAVE_STRINGS_H */ #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include "src/util/crc.h" #if (ALIGNOF_LONG == 8) # define PRTE_CRC_WORD_MASK_ 0x7 #elif (ALIGNOF_LONG == 4) # define PRTE_CRC_WORD_MASK_ 0x3 #else # define PRTE_CRC_WORD_MASK_ 0xFFFF #endif #define WORDALIGNED(v) (((intptr_t) v & PRTE_CRC_WORD_MASK_) ? false : true) #define INTALIGNED(v) (((intptr_t) v & 3) ? false : true) /* * this version of bcopy_csum() looks a little too long, but it * handles cumulative checksumming for arbitrary lengths and address * alignments as best as it can; the contents of lastPartialLong and * lastPartialLength are updated to reflected the last partial word's * value and length (in bytes) -- this should allow proper handling of * checksumming contiguous or noncontiguous buffers via multiple calls * of bcopy_csum() - Mitch */ unsigned long prte_bcopy_csum_partial(const void *source, void *destination, size_t copylen, size_t csumlen, unsigned long *lastPartialLong, size_t *lastPartialLength) { unsigned long *src = (unsigned long *) source; unsigned long *dest = (unsigned long *) destination; unsigned long csum = 0; size_t csumlenresidue; unsigned long i, temp; csumlenresidue = (csumlen > copylen) ? (csumlen - copylen) : 0; temp = *lastPartialLong; if (WORDALIGNED(source) && WORDALIGNED(dest)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); dest = (unsigned long *) ((char *) dest + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); copylen -= sizeof(unsigned long) - *lastPartialLength; /* now we have an unaligned source and an unaligned destination */ for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialLong = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned long *) ((char *) src + copylen); dest = (unsigned long *) ((char *) dest + copylen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += copylen; copylen = 0; } } else { /* fast path... */ size_t numLongs = copylen / sizeof(unsigned long); for (i = 0; i < numLongs; i++) { csum += *src; *dest++ = *src++; } *lastPartialLong = 0; *lastPartialLength = 0; if (WORDALIGNED(copylen) && (csumlenresidue == 0)) { return (csum); } else { copylen -= i * sizeof(unsigned long); } } } else if (WORDALIGNED(source)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); dest = (unsigned long *) ((char *) dest + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); copylen -= sizeof(unsigned long) - *lastPartialLength; /* now we have an unaligned source and an unknown alignment for our destination */ if (WORDALIGNED(dest)) { size_t numLongs = copylen / sizeof(unsigned long); for (i = 0; i < numLongs; i++) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; *dest++ = temp; } copylen -= i * sizeof(unsigned long); } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } } *lastPartialLong = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned long *) ((char *) src + copylen); dest = (unsigned long *) ((char *) dest + copylen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { temp = *src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLong = 0; *lastPartialLength = 0; } } else if (WORDALIGNED(dest)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); dest = (unsigned long *) ((char *) dest + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); copylen -= sizeof(unsigned long) - *lastPartialLength; /* now we have a source of unknown alignment and a unaligned destination */ if (WORDALIGNED(src)) { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { temp = *src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLong = 0; *lastPartialLength = 0; } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialLong = 0; } } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned long *) ((char *) src + copylen); dest = (unsigned long *) ((char *) dest + copylen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; *dest++ = temp; } *lastPartialLength = 0; *lastPartialLong = 0; } } else { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); dest = (unsigned long *) ((char *) dest + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); copylen -= sizeof(unsigned long) - *lastPartialLength; /* now we have an unknown alignment for our source and destination */ if (WORDALIGNED(src) && WORDALIGNED(dest)) { size_t numLongs = copylen / sizeof(unsigned long); for (i = 0; i < numLongs; i++) { csum += *src; *dest++ = *src++; } copylen -= i * sizeof(unsigned long); } else { /* safe but slower for all other alignments */ for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } } *lastPartialLong = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned long *) ((char *) src + copylen); dest = (unsigned long *) ((char *) dest + copylen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialLong = 0; } } /* if copylen is non-zero there was a bit left, less than an unsigned long's worth */ if ((copylen != 0) && (csumlenresidue == 0)) { temp = *lastPartialLong; if (*lastPartialLength) { if (copylen >= (sizeof(unsigned long) - *lastPartialLength)) { /* copy all remaining bytes from src to dest */ unsigned long copytemp = 0; memcpy(©temp, src, copylen); memcpy(dest, ©temp, copylen); /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += (temp - *lastPartialLong); copylen -= sizeof(unsigned long) - *lastPartialLength; src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); *lastPartialLength = copylen; /* reset temp, and calculate next partial word */ temp = 0; if (copylen) { memcpy(&temp, src, copylen); } /* add it to the the checksum */ csum += temp; *lastPartialLong = temp; } else { /* copy all remaining bytes from src to dest */ unsigned long copytemp = 0; memcpy(©temp, src, copylen); memcpy(dest, ©temp, copylen); /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += temp - *lastPartialLong; *lastPartialLong = temp; *lastPartialLength += copylen; } } else { /* fast path... */ /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */ memcpy(&temp, src, copylen); csum += temp; memcpy(dest, &temp, copylen); *lastPartialLong = temp; *lastPartialLength = copylen; /* done...return the checksum */ } } else if (csumlenresidue != 0) { if (copylen != 0) { temp = 0; memcpy(&temp, src, copylen); memcpy(dest, &temp, copylen); } if (csumlenresidue < (sizeof(unsigned long) - copylen - *lastPartialLength)) { temp = *lastPartialLong; memcpy(((char *) &temp + *lastPartialLength), src, (copylen + csumlenresidue)); /* avoid unsigned arithmetic overflow by subtracting the old partial */ /* word from the new one before adding to the checksum... */ csum += temp - *lastPartialLong; src++; *lastPartialLong = temp; *lastPartialLength += copylen + csumlenresidue; csumlenresidue = 0; } else { /* we have enough chksum data to fill out our last partial */ /* word */ temp = *lastPartialLong; memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); /* avoid unsigned arithmetic overflow by subtracting the old partial */ /* word from the new one before adding to the checksum... */ csum += temp - *lastPartialLong; src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); csumlenresidue -= sizeof(unsigned long) - *lastPartialLength - copylen; *lastPartialLength = 0; *lastPartialLong = 0; } if (WORDALIGNED(src)) { for (i = 0; i < csumlenresidue / sizeof(unsigned long); i++) { csum += *src++; } } else { for (i = 0; i < csumlenresidue / sizeof(unsigned long); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } } csumlenresidue -= i * sizeof(unsigned long); if (csumlenresidue) { temp = 0; memcpy(&temp, src, csumlenresidue); csum += temp; *lastPartialLong = temp; *lastPartialLength = csumlenresidue; } } /* end else if (csumlenresidue != 0) */ return csum; } unsigned int prte_bcopy_uicsum_partial(const void *source, void *destination, size_t copylen, size_t csumlen, unsigned int *lastPartialInt, size_t *lastPartialLength) { unsigned int *src = (unsigned int *) source; unsigned int *dest = (unsigned int *) destination; unsigned int csum = 0; size_t csumlenresidue; unsigned long i; unsigned int temp; csumlenresidue = (csumlen > copylen) ? (csumlen - copylen) : 0; temp = *lastPartialInt; if (INTALIGNED(source) && INTALIGNED(dest)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); copylen -= sizeof(unsigned int) - *lastPartialLength; /* now we have an unaligned source and an unaligned destination */ for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialInt = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned int *) ((char *) src + copylen); dest = (unsigned int *) ((char *) dest + copylen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += copylen; copylen = 0; } } else { /* fast path... */ size_t numLongs = copylen / sizeof(unsigned int); for (i = 0; i < numLongs; i++) { csum += *src; *dest++ = *src++; } *lastPartialInt = 0; *lastPartialLength = 0; if (INTALIGNED(copylen) && (csumlenresidue == 0)) { return (csum); } else { copylen -= i * sizeof(unsigned int); } } } else if (INTALIGNED(source)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); copylen -= sizeof(unsigned int) - *lastPartialLength; /* now we have an unaligned source and an unknown alignment for our destination */ if (INTALIGNED(dest)) { size_t numLongs = copylen / sizeof(unsigned int); for (i = 0; i < numLongs; i++) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; *dest++ = temp; } copylen -= i * sizeof(unsigned int); } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } } *lastPartialInt = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned int *) ((char *) src + copylen); dest = (unsigned int *) ((char *) dest + copylen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { temp = *src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialInt = 0; *lastPartialLength = 0; } } else if (INTALIGNED(dest)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); copylen -= sizeof(unsigned int) - *lastPartialLength; /* now we have a source of unknown alignment and a unaligned destination */ if (INTALIGNED(src)) { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { temp = *src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialInt = 0; *lastPartialLength = 0; } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialInt = 0; } } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned int *) ((char *) src + copylen); dest = (unsigned int *) ((char *) dest + copylen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; *dest++ = temp; } *lastPartialLength = 0; *lastPartialInt = 0; } } else { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); memcpy(dest, ((char *) &temp + *lastPartialLength), (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); dest = (unsigned int *) ((char *) dest + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); copylen -= sizeof(unsigned int) - *lastPartialLength; /* now we have an unknown alignment for our source and destination */ if (INTALIGNED(src) && INTALIGNED(dest)) { size_t numLongs = copylen / sizeof(unsigned int); for (i = 0; i < numLongs; i++) { csum += *src; *dest++ = *src++; } copylen -= i * sizeof(unsigned int); } else { /* safe but slower for all other alignments */ for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } } *lastPartialInt = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); memcpy(dest, ((char *) &temp + *lastPartialLength), copylen); src = (unsigned int *) ((char *) src + copylen); dest = (unsigned int *) ((char *) dest + copylen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += copylen; copylen = 0; } } else { for (; copylen >= sizeof(*src); copylen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; memcpy(dest, &temp, sizeof(temp)); dest++; } *lastPartialLength = 0; *lastPartialInt = 0; } } /* if copylen is non-zero there was a bit left, less than an unsigned int's worth */ if ((copylen != 0) && (csumlenresidue == 0)) { temp = *lastPartialInt; if (*lastPartialLength) { if (copylen >= (sizeof(unsigned int) - *lastPartialLength)) { /* copy all remaining bytes from src to dest */ unsigned int copytemp = 0; memcpy(©temp, src, copylen); memcpy(dest, ©temp, copylen); /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += (temp - *lastPartialInt); copylen -= sizeof(unsigned int) - *lastPartialLength; src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); *lastPartialLength = copylen; /* reset temp, and calculate next partial word */ temp = 0; if (copylen) { memcpy(&temp, src, copylen); } /* add it to the the checksum */ csum += temp; *lastPartialInt = temp; } else { /* copy all remaining bytes from src to dest */ unsigned int copytemp = 0; memcpy(©temp, src, copylen); memcpy(dest, ©temp, copylen); /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, copylen); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += temp - *lastPartialInt; *lastPartialInt = temp; *lastPartialLength += copylen; } } else { /* fast path... */ /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */ memcpy(&temp, src, copylen); csum += temp; memcpy(dest, &temp, copylen); *lastPartialInt = temp; *lastPartialLength = copylen; /* done...return the checksum */ } } else if (csumlenresidue != 0) { if (copylen != 0) { temp = 0; memcpy(&temp, src, copylen); memcpy(dest, &temp, copylen); } if (csumlenresidue < (sizeof(unsigned int) - copylen - *lastPartialLength)) { temp = *lastPartialInt; memcpy(((char *) &temp + *lastPartialLength), src, (copylen + csumlenresidue)); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += temp - *lastPartialInt; src++; *lastPartialInt = temp; *lastPartialLength += copylen + csumlenresidue; csumlenresidue = 0; } else { /* we have enough chksum data to fill out our last partial * word */ temp = *lastPartialInt; memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); /* avoid unsigned arithmetic overflow by subtracting the old partial * word from the new one before adding to the checksum... */ csum += temp - *lastPartialInt; src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); csumlenresidue -= sizeof(unsigned int) - *lastPartialLength - copylen; *lastPartialLength = 0; *lastPartialInt = 0; } if (INTALIGNED(src)) { for (i = 0; i < csumlenresidue / sizeof(unsigned int); i++) { csum += *src++; } } else { for (i = 0; i < csumlenresidue / sizeof(unsigned int); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } } csumlenresidue -= i * sizeof(unsigned int); if (csumlenresidue) { temp = 0; memcpy(&temp, src, csumlenresidue); csum += temp; *lastPartialInt = temp; *lastPartialLength = csumlenresidue; } } /* end else if (csumlenresidue != 0) */ return csum; } /* * csum() generates a bcopy_csum() - compatible checksum that can be * called multiple times */ unsigned long prte_csum_partial(const void *source, size_t csumlen, unsigned long *lastPartialLong, size_t *lastPartialLength) { unsigned long *src = (unsigned long *) source; unsigned long csum = 0; unsigned long i, temp; temp = *lastPartialLong; if (WORDALIGNED(source)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); csumlen -= sizeof(unsigned long) - *lastPartialLength; /* now we have an unaligned source */ for (i = 0; i < csumlen / sizeof(unsigned long); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } csumlen -= i * sizeof(unsigned long); *lastPartialLong = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); src = (unsigned long *) ((char *) src + csumlen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += csumlen; csumlen = 0; } } else { /* fast path... */ size_t numLongs = csumlen / sizeof(unsigned long); for (i = 0; i < numLongs; i++) { csum += *src++; } *lastPartialLong = 0; *lastPartialLength = 0; if (WORDALIGNED(csumlen)) { return (csum); } else { csumlen -= i * sizeof(unsigned long); } } } else { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); csum += (temp - *lastPartialLong); csumlen -= sizeof(unsigned long) - *lastPartialLength; /* now we have a source of unknown alignment */ if (WORDALIGNED(src)) { for (i = 0; i < csumlen / sizeof(unsigned long); i++) { csum += *src++; } csumlen -= i * sizeof(unsigned long); *lastPartialLong = 0; *lastPartialLength = 0; } else { for (i = 0; i < csumlen / sizeof(unsigned long); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } csumlen -= i * sizeof(unsigned long); *lastPartialLong = 0; *lastPartialLength = 0; } } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); src = (unsigned long *) ((char *) src + csumlen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += csumlen; csumlen = 0; } } else { for (; csumlen >= sizeof(*src); csumlen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; } *lastPartialLength = 0; *lastPartialLong = 0; } } /* if csumlen is non-zero there was a bit left, less than an unsigned long's worth */ if (csumlen != 0) { temp = *lastPartialLong; if (*lastPartialLength) { if (csumlen >= (sizeof(unsigned long) - *lastPartialLength)) { /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned long) - *lastPartialLength)); csum += (temp - *lastPartialLong); csumlen -= sizeof(unsigned long) - *lastPartialLength; src = (unsigned long *) ((char *) src + sizeof(unsigned long) - *lastPartialLength); *lastPartialLength = csumlen; /* reset temp, and calculate next partial word */ temp = 0; if (csumlen) { memcpy(&temp, src, csumlen); } /* add it to the the checksum */ csum += temp; *lastPartialLong = temp; } else { /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); csum += (temp - *lastPartialLong); *lastPartialLong = temp; *lastPartialLength += csumlen; } } else { /* fast path... */ /* temp and *lastPartialLong are 0 if *lastPartialLength is 0... */ memcpy(&temp, src, csumlen); csum += temp; *lastPartialLong = temp; *lastPartialLength = csumlen; /* done...return the checksum */ } } return csum; } unsigned int prte_uicsum_partial(const void *source, size_t csumlen, unsigned int *lastPartialInt, size_t *lastPartialLength) { unsigned int *src = (unsigned int *) source; unsigned int csum = 0; unsigned int temp; unsigned long i; temp = *lastPartialInt; if (INTALIGNED(source)) { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); csumlen -= sizeof(unsigned int) - *lastPartialLength; /* now we have an unaligned source */ for (i = 0; i < csumlen / sizeof(unsigned int); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } csumlen -= i * sizeof(unsigned int); *lastPartialInt = 0; *lastPartialLength = 0; } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); src = (unsigned int *) ((char *) src + csumlen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += csumlen; csumlen = 0; } } else { /* fast path... */ size_t numLongs = csumlen / sizeof(unsigned int); for (i = 0; i < numLongs; i++) { csum += *src++; } *lastPartialInt = 0; *lastPartialLength = 0; if (INTALIGNED(csumlen)) { return (csum); } else { csumlen -= i * sizeof(unsigned int); } } } else { if (*lastPartialLength) { /* do we have enough data to fill out the partial word? */ if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* YES, we do... */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); csum += (temp - *lastPartialInt); csumlen -= sizeof(unsigned int) - *lastPartialLength; /* now we have a source of unknown alignment */ if (INTALIGNED(src)) { for (i = 0; i < csumlen / sizeof(unsigned int); i++) { csum += *src++; } csumlen -= i * sizeof(unsigned int); *lastPartialInt = 0; *lastPartialLength = 0; } else { for (i = 0; i < csumlen / sizeof(unsigned int); i++) { memcpy(&temp, src, sizeof(temp)); csum += temp; src++; } csumlen -= i * sizeof(unsigned int); *lastPartialInt = 0; *lastPartialLength = 0; } } else { /* NO, we don't... */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); src = (unsigned int *) ((char *) src + csumlen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += csumlen; csumlen = 0; } } else { for (; csumlen >= sizeof(*src); csumlen -= sizeof(*src)) { memcpy(&temp, src, sizeof(temp)); src++; csum += temp; } *lastPartialLength = 0; *lastPartialInt = 0; } } /* if csumlen is non-zero there was a bit left, less than an unsigned int's worth */ if (csumlen != 0) { temp = *lastPartialInt; if (*lastPartialLength) { if (csumlen >= (sizeof(unsigned int) - *lastPartialLength)) { /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, (sizeof(unsigned int) - *lastPartialLength)); csum += (temp - *lastPartialInt); csumlen -= sizeof(unsigned int) - *lastPartialLength; src = (unsigned int *) ((char *) src + sizeof(unsigned int) - *lastPartialLength); *lastPartialLength = csumlen; /* reset temp, and calculate next partial word */ temp = 0; if (csumlen) { memcpy(&temp, src, csumlen); } /* add it to the the checksum */ csum += temp; *lastPartialInt = temp; } else { /* fill out rest of partial word and add to checksum */ memcpy(((char *) &temp + *lastPartialLength), src, csumlen); csum += (temp - *lastPartialInt); *lastPartialInt = temp; *lastPartialLength += csumlen; } } else { /* fast path... */ /* temp and *lastPartialInt are 0 if *lastPartialLength is 0... */ memcpy(&temp, src, csumlen); csum += temp; *lastPartialInt = temp; *lastPartialLength = csumlen; /* done...return the checksum */ } } return csum; } /* globals for CRC32 bcopy and calculation routines */ static bool _prte_crc_table_initialized = false; static unsigned int _prte_crc_table[256]; /* CRC32 table generation routine - thanks to Charles Michael Heard for his * optimized CRC32 code... */ void prte_initialize_crc_table(void) { register int i, j; register unsigned int crc_accum; for (i = 0; i < 256; i++) { crc_accum = (i << 24); for (j = 0; j < 8; j++) { if (crc_accum & 0x80000000) crc_accum = (crc_accum << 1) ^ CRC_POLYNOMIAL; else crc_accum = (crc_accum << 1); } _prte_crc_table[i] = crc_accum; } /* set global bool to true to do this work once! */ _prte_crc_table_initialized = true; return; } unsigned int prte_bcopy_uicrc_partial(const void *source, void *destination, size_t copylen, size_t crclen, unsigned int partial_crc) { size_t crclenresidue = (crclen > copylen) ? (crclen - copylen) : 0; register int i, j; register unsigned char t; unsigned int tmp; if (!_prte_crc_table_initialized) { prte_initialize_crc_table(); } if (INTALIGNED(source) && INTALIGNED(destination)) { register unsigned int *src = (unsigned int *) source; register unsigned int *dst = (unsigned int *) destination; register unsigned char *ts, *td; /* copy whole integers */ while (copylen >= sizeof(unsigned int)) { tmp = *src++; *dst++ = tmp; ts = (unsigned char *) &tmp; for (j = 0; j < (int) sizeof(unsigned int); j++) { i = ((partial_crc >> 24) ^ *ts++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } copylen -= sizeof(unsigned int); } ts = (unsigned char *) src; td = (unsigned char *) dst; /* copy partial integer */ while (copylen--) { t = *ts++; *td++ = t; i = ((partial_crc >> 24) ^ t) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } /* calculate CRC over remaining bytes... */ while (crclenresidue--) { i = ((partial_crc >> 24) ^ *ts++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } } else { register unsigned char *src = (unsigned char *) source; register unsigned char *dst = (unsigned char *) destination; while (copylen--) { t = *src++; *dst++ = t; i = ((partial_crc >> 24) ^ t) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } while (crclenresidue--) { i = ((partial_crc >> 24) ^ *src++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } } return partial_crc; } unsigned int prte_uicrc_partial(const void *source, size_t crclen, unsigned int partial_crc) { register int i, j; register unsigned char *t; unsigned int tmp; if (!_prte_crc_table_initialized) { prte_initialize_crc_table(); } if (INTALIGNED(source)) { register unsigned int *src = (unsigned int *) source; while (crclen >= sizeof(unsigned int)) { tmp = *src++; t = (unsigned char *) &tmp; for (j = 0; j < (int) sizeof(unsigned int); j++) { i = ((partial_crc >> 24) ^ *t++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } crclen -= sizeof(unsigned int); } t = (unsigned char *) src; while (crclen--) { i = ((partial_crc >> 24) ^ *t++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } } else { register unsigned char *src = (unsigned char *) source; while (crclen--) { i = ((partial_crc >> 24) ^ *src++) & 0xff; partial_crc = (partial_crc << 8) ^ _prte_crc_table[i]; } } return partial_crc; } prrte-4.1.1~rc2/src/util/dash_host/0000775000175000017500000000000015203542132017357 5ustar alastairalastairprrte-4.1.1~rc2/src/util/dash_host/dash_host.c0000664000175000017500000007201315203542132021502 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #include "constants.h" #include "types.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/plm/plm_types.h" #include "src/mca/ras/base/base.h" #include "src/runtime/prte_globals.h" #include "dash_host.h" int prte_util_dash_host_compute_slots(prte_node_t *node, char *hosts) { char **specs, *cptr; int slots = 0; int n; specs = PMIx_Argv_split(hosts, ','); /* see if this node appears in the list */ for (n = 0; NULL != specs[n]; n++) { /* check if the #slots was specified */ if (NULL != (cptr = strchr(specs[n], ':'))) { *cptr = '\0'; ++cptr; } else { cptr = NULL; } if (prte_quickmatch(node, specs[n])) { if (NULL != cptr) { if ('*' == *cptr || 0 == strcmp(cptr, "auto")) { slots += node->slots - node->slots_inuse; } else { slots += strtol(cptr, NULL, 10); } } else { ++slots; } } } PMIx_Argv_free(specs); return slots; } /* we can only enter this routine if no other allocation * was found, so we only need to know that finding any * relative node syntax should generate an immediate error */ int prte_util_add_dash_host_nodes(pmix_list_t *nodes, char *hosts, bool allocating) { pmix_list_item_t *item; int32_t i, j, k; int rc, nodeidx; char **host_argv = NULL; char **mapped_nodes = NULL, **mini_map, *ndname; prte_node_t *node, *nd; pmix_list_t adds; bool needcheck; int slots = 0; bool slots_given; char *cptr; char *shortname; char *rawname; bool add_slots = false; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: parsing args %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); PMIX_CONSTRUCT(&adds, pmix_list_t); host_argv = PMIx_Argv_split(hosts, ','); if (0 < pmix_list_get_size(nodes)) { needcheck = true; } else { needcheck = false; } /* Accumulate all of the host name mappings */ for (j = 0; j < PMIx_Argv_count(host_argv); ++j) { mini_map = PMIx_Argv_split(host_argv[j], ','); if (mapped_nodes == NULL) { mapped_nodes = mini_map; } else { for (k = 0; NULL != mini_map[k]; ++k) { rc = PMIx_Argv_append_nosize(&mapped_nodes, mini_map[k]); if (PRTE_SUCCESS != rc) { PMIx_Argv_free(host_argv); PMIx_Argv_free(mini_map); goto cleanup; } } PMIx_Argv_free(mini_map); } } PMIx_Argv_free(host_argv); mini_map = NULL; /* Did we find anything? If not, then do nothing */ if (NULL == mapped_nodes) { rc = PRTE_SUCCESS; goto cleanup; } for (i = 0; NULL != mapped_nodes[i]; ++i) { /* if the specified node contains a relative node syntax, * and we are allocating, then ignore it */ if ('+' == mapped_nodes[i][0]) { if (!allocating) { if ('e' == mapped_nodes[i][1] || 'E' == mapped_nodes[i][1]) { /* request for empty nodes - do they want * all of them? */ if (NULL != (cptr = strchr(mapped_nodes[i], ':'))) { /* the colon indicates a specific # are requested */ ++cptr; j = strtoul(cptr, NULL, 10); } else if ('\0' != mapped_nodes[0][2]) { j = strtoul(&mapped_nodes[0][2], NULL, 10); } else { /* add them all */ j = prte_node_pool->size; } for (k = 0; 0 < j && k < prte_node_pool->size; k++) { if (NULL != (node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, k))) { if (0 == node->num_procs) { PMIx_Argv_append_nosize(&mini_map, node->name); --j; } } } } else if ('n' == mapped_nodes[i][1] || 'N' == mapped_nodes[i][1]) { /* they want a specific relative node #, so * look it up on global pool */ if ('\0' == mapped_nodes[i][2]) { /* they forgot to tell us the # */ pmix_show_help("help-dash-host.txt", "dash-host:invalid-relative-node-syntax", true, mapped_nodes[i]); rc = PRTE_ERR_SILENT; goto cleanup; } nodeidx = strtol(&mapped_nodes[i][2], NULL, 10); if (nodeidx < 0 || nodeidx > (int) prte_node_pool->size) { /* this is an error */ pmix_show_help("help-dash-host.txt", "dash-host:relative-node-out-of-bounds", true, nodeidx, mapped_nodes[i]); rc = PRTE_ERR_SILENT; goto cleanup; } /* if the HNP is not allocated, then we need to * adjust the index as the node pool is offset * by one */ if (!prte_hnp_is_allocated) { nodeidx++; } /* see if that location is filled */ node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, nodeidx); if (NULL == node) { /* this is an error */ pmix_show_help("help-dash-host.txt", "dash-host:relative-node-not-found", true, nodeidx, mapped_nodes[i]); rc = PRTE_ERR_SILENT; goto cleanup; } /* add this node to the list */ PMIx_Argv_append_nosize(&mini_map, node->name); } else { /* invalid relative node syntax */ pmix_show_help("help-dash-host.txt", "dash-host:invalid-relative-node-syntax", true, mapped_nodes[i]); rc = PRTE_ERR_SILENT; goto cleanup; } } } else { /* just one node was given */ PMIx_Argv_append_nosize(&mini_map, mapped_nodes[i]); } } if (NULL == mini_map) { rc = PRTE_SUCCESS; goto cleanup; } /* go through the names found and add them to the host list. If they're not unique, then bump the slots count for each duplicate */ for (i = 0; NULL != mini_map[i]; i++) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: working node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), mini_map[i])); /* see if the node contains the number of slots */ slots_given = false; if (NULL != (cptr = strchr(mini_map[i], ':'))) { *cptr = '\0'; ++cptr; if ('*' == *cptr || 0 == strcmp(cptr, "auto")) { /* auto-detect #slots */ slots = -1; slots_given = false; } else { slots = strtol(cptr, NULL, 10); if ('+' == *cptr || '-' == *cptr) { // mark that we are being asked to increase/decrease #slots add_slots = true; } slots_given = true; } } /* check for local name and compute non-fqdn name */ shortname = NULL; rawname = NULL; if (prte_check_host_is_local(mini_map[i])) { ndname = prte_process_info.nodename; } else { ndname = mini_map[i]; } if (!prte_keep_fqdn_hostnames) { // Strip off the FQDN if present, ignore IP addresses if (!pmix_net_isaddr(mini_map[i])) { cptr = strchr(ndname, '.'); if (NULL != cptr) { rawname = strdup(ndname); *cptr = '\0'; shortname = strdup(ndname); *cptr = '.'; } } } /* see if a node of this name is already on the list */ node = prte_node_match(&adds, ndname); if (NULL == node && NULL != shortname) { node = prte_node_match(&adds, shortname); } if (NULL != node) { if (slots_given) { node->slots += slots; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); if (add_slots) { prte_set_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } else if (slots < 0) { node->slots = 0; PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } else { ++node->slots; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: node %s already on list - slots %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, node->slots)); if (NULL != shortname) { free(shortname); shortname = NULL; } if (NULL != rawname) { node->rawname = rawname; rawname = NULL; } } else { /* if we didn't find it, add it to the list */ node = PMIX_NEW(prte_node_t); if (NULL == node) { PMIx_Argv_free(mapped_nodes); if (NULL != shortname) { free(shortname); } if (NULL != rawname) { free(rawname); } return PRTE_ERR_OUT_OF_RESOURCE; } if (prte_keep_fqdn_hostnames || NULL == shortname) { node->name = strdup(ndname); } else { node->name = strdup(shortname); } if (NULL != rawname) { node->rawname = rawname; rawname = NULL; } PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: added node %s to list - slots %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, slots)); node->state = PRTE_NODE_STATE_UP; node->slots_inuse = 0; node->slots_max = 0; if (slots_given) { node->slots = slots; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); if (add_slots) { prte_set_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } else if (slots < 0) { node->slots = 0; PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } else { node->slots = 1; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } pmix_list_append(&adds, &node->super); } if (0 != strcmp(node->name, mini_map[i])) { // add the mini_map name to the list of aliases PMIx_Argv_append_unique_nosize(&node->aliases, mini_map[i]); } // ensure the non-fqdn version is saved if (NULL != shortname && 0 != strcmp(shortname, node->name)) { PMIx_Argv_append_unique_nosize(&node->aliases, shortname); } if (NULL != shortname) { free(shortname); } if (NULL != rawname) { free(rawname); } } PMIx_Argv_free(mini_map); /* transfer across all unique nodes */ while (NULL != (item = pmix_list_remove_first(&adds))) { nd = (prte_node_t *) item; if (needcheck) { node = prte_node_match(nodes, nd->name); if (NULL != node) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: found existing node %s on input list - adding slots", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); if (PRTE_FLAG_TEST(nd, PRTE_NODE_FLAG_SLOTS_GIVEN)) { /* transfer across the number of slots */ node->slots += nd->slots; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } PMIX_RELEASE(item); } else { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: adding node %s with %d slots to final list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), nd->name, nd->slots)); pmix_list_append(nodes, &nd->super); } } else { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s dashhost: adding node %s with %d slots to final list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), nd->name, nd->slots)); pmix_list_append(nodes, &nd->super); } } if (prte_managed_allocation && !allocating) { prte_node_t *node_from_pool = NULL; PMIX_LIST_FOREACH(node, nodes, prte_node_t) { needcheck = true; for (i = 0; i < prte_node_pool->size; i++) { node_from_pool = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, i); if (NULL == node_from_pool) { continue; } if (prte_nptr_match(node_from_pool, node)) { needcheck = false; if (node->slots < node_from_pool->slots) { node_from_pool->slots = node->slots; } break; } } if (needcheck) { // node in -host was not in allocation - this is not allowed pmix_show_help("help-dash-host.txt", "not-all-mapped-alloc", true, node->name); rc = PRTE_ERR_SILENT; goto cleanup; } } } rc = PRTE_SUCCESS; cleanup: if (NULL != mapped_nodes) { PMIx_Argv_free(mapped_nodes); } PMIX_LIST_DESTRUCT(&adds); return rc; } /* the -host option can always be used in both absolute * and relative mode, so we have to check for pre-existing * allocations if we are to use relative node syntax */ static int parse_dash_host(char ***mapped_nodes, char *hosts) { int32_t j, k, start; int rc = PRTE_SUCCESS; char **mini_map = NULL, *cptr; int nodeidx, nnodes, p; prte_node_t *node; char **host_argv = NULL; host_argv = PMIx_Argv_split(hosts, ','); if (prte_hnp_is_allocated) { start = 0; } else { start = 1; } /* Accumulate all of the host name mappings */ for (j = 0; j < PMIx_Argv_count(host_argv); ++j) { mini_map = PMIx_Argv_split(host_argv[j], ','); for (k = 0; NULL != mini_map[k]; ++k) { if ('+' == mini_map[k][0]) { /* see if we specified empty nodes */ if ('e' == mini_map[k][1] || 'E' == mini_map[k][1]) { /* request for empty nodes - do they want * all of them? */ if (NULL != (cptr = strchr(mini_map[k], ':'))) { /* the colon indicates a specific # are requested */ ++cptr; if ('\0' == *cptr) { // missing number of nodes being requested pmix_show_help("help-dash-host.txt", "dash-host:invalid-relative-node-syntax", true, mini_map[k]); rc = PRTE_ERR_SILENT; goto cleanup; } nnodes = strtol(cptr, NULL, 10); for (j=start, p=0; j < (int32_t)prte_node_pool->size && p < nnodes; j++) { node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, j); if (NULL == node) { continue; } // if the node is empty, capture it if (0 == node->num_procs) { PMIx_Argv_append_nosize(mapped_nodes, node->name); ++p; } } if (p < nnodes) { // not enough empty nodes pmix_show_help("help-dash-host.txt", "dash-host:not-enough-empty", true, nnodes-p); rc = PRTE_ERR_SILENT; goto cleanup; } } else { /* add a marker to the list */ PMIx_Argv_append_nosize(mapped_nodes, "*"); } } else if ('n' == mini_map[k][1] || 'N' == mini_map[k][1]) { /* they want a specific relative node #, so * look it up on global pool */ nodeidx = strtol(&mini_map[k][2], NULL, 10); if (nodeidx < 0 || nodeidx > (int) prte_node_pool->size) { /* this is an error */ pmix_show_help("help-dash-host.txt", "dash-host:relative-node-out-of-bounds", true, nodeidx, mini_map[k]); rc = PRTE_ERR_SILENT; goto cleanup; } /* if the HNP is not allocated, then we need to * adjust the index as the node pool is offset * by one */ if (!prte_hnp_is_allocated) { nodeidx++; } /* see if that location is filled */ node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, nodeidx); if (NULL == node) { /* this is an error */ pmix_show_help("help-dash-host.txt", "dash-host:relative-node-not-found", true, nodeidx, mini_map[k]); rc = PRTE_ERR_SILENT; goto cleanup; } /* add this node to the list */ PMIx_Argv_append_nosize(mapped_nodes, node->name); } else { /* invalid relative node syntax */ pmix_show_help("help-dash-host.txt", "dash-host:invalid-relative-node-syntax", true, mini_map[k]); rc = PRTE_ERR_SILENT; goto cleanup; } } else { /* non-relative syntax - add to list */ /* remove any modifier */ if (NULL != (cptr = strchr(mini_map[k], ':'))) { *cptr = '\0'; } /* check for local alias */ if (prte_check_host_is_local(mini_map[k])) { PMIx_Argv_append_nosize(mapped_nodes, prte_process_info.nodename); } else { PMIx_Argv_append_nosize(mapped_nodes, mini_map[k]); } } } PMIx_Argv_free(mini_map); mini_map = NULL; } cleanup: if (NULL != host_argv) { PMIx_Argv_free(host_argv); } if (NULL != mini_map) { PMIx_Argv_free(mini_map); } return rc; } int prte_util_filter_dash_host_nodes(pmix_list_t *nodes, char *hosts, bool remove) { pmix_list_item_t *item; pmix_list_item_t *next; int32_t i, j, len_mapped_node = 0; int rc, test; char **mapped_nodes = NULL; prte_node_t *node; int num_empty = 0; pmix_list_t keep; bool want_all_empty = false; char *cptr; size_t lst, lmn; /* if the incoming node list is empty, then there * is nothing to filter! */ if (pmix_list_is_empty(nodes)) { return PRTE_SUCCESS; } if (PRTE_SUCCESS != (rc = parse_dash_host(&mapped_nodes, hosts))) { PRTE_ERROR_LOG(rc); return rc; } /* Did we find anything? If not, then do nothing */ if (NULL == mapped_nodes) { return PRTE_SUCCESS; } /* NOTE: The following logic is based on knowing that * any node can only be included on the incoming * nodes list ONCE. */ len_mapped_node = PMIx_Argv_count(mapped_nodes); /* setup a working list so we can put the final list * of nodes in order. This way, if the user specifies a * set of nodes, we will use them in the order in which * they were specifed. Note that empty node requests * will always be appended to the end */ PMIX_CONSTRUCT(&keep, pmix_list_t); for (i = 0; i < len_mapped_node; ++i) { /* check if we are supposed to add some number of empty * nodes here */ if ('*' == mapped_nodes[i][0]) { /* if there is a number after the '*', then we are * to insert a specific # of nodes */ if ('\0' == mapped_nodes[i][1]) { /* take all empty nodes from the list */ num_empty = INT_MAX; want_all_empty = true; } else { /* extract number of nodes to take */ num_empty = strtol(&mapped_nodes[i][1], NULL, 10); } /* search for empty nodes and take them */ item = pmix_list_get_first(nodes); while (0 < num_empty && item != pmix_list_get_end(nodes)) { next = pmix_list_get_next(item); /* save this position */ node = (prte_node_t *) item; /* see if this node is empty */ if (0 == node->slots_inuse) { /* check to see if it is specified later */ for (j = i + 1; j < len_mapped_node; j++) { if (0 == strcmp(mapped_nodes[j], node->name)) { /* specified later - skip this one */ goto skipnode; } } if (remove) { /* remove item from list */ pmix_list_remove_item(nodes, item); /* xfer to keep list */ pmix_list_append(&keep, item); } else { /* mark the node as found */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_MAPPED); } --num_empty; } skipnode: item = next; } } else { /* remove any modifier */ if (NULL != (cptr = strchr(mapped_nodes[i], ':'))) { *cptr = '\0'; } /* we are looking for a specific node on the list. */ cptr = NULL; lmn = strtoul(mapped_nodes[i], &cptr, 10); item = pmix_list_get_first(nodes); while (item != pmix_list_get_end(nodes)) { next = pmix_list_get_next(item); /* save this position */ node = (prte_node_t *) item; /* search -host list to see if this one is found */ if (prte_managed_allocation && (NULL == cptr || 0 == strlen(cptr))) { /* if we are only given a number, then we test the * value against the number in the node name. This allows support for * launch_id-based environments. For example, a hostname * of "nid0015" can be referenced by "--host 15" */ for (j = strlen(node->name) - 1; 0 < j; j--) { if (!isdigit(node->name[j])) { j++; break; } } if (j >= (int) (strlen(node->name) - 1)) { test = 0; } else { lst = strtoul(&node->name[j], NULL, 10); test = (lmn == lst) ? 0 : 1; } } else { test = (prte_quickmatch(node, mapped_nodes[i])) ? 0 : 1; } if (0 == test) { if (remove) { /* remove item from list */ pmix_list_remove_item(nodes, item); /* xfer to keep list */ pmix_list_append(&keep, item); } else { /* mark the node as found */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_MAPPED); } break; } item = next; } } /* done with the mapped entry */ free(mapped_nodes[i]); mapped_nodes[i] = NULL; } /* was something specified that was -not- found? */ for (i = 0; i < len_mapped_node; i++) { if (NULL != mapped_nodes[i]) { pmix_show_help("help-dash-host.txt", "not-all-mapped-alloc", true, mapped_nodes[i]); rc = PRTE_ERR_SILENT; goto cleanup; } } if (!remove) { /* all done */ rc = PRTE_SUCCESS; goto cleanup; } /* clear the rest of the nodes list */ while (NULL != (item = pmix_list_remove_first(nodes))) { PMIX_RELEASE(item); } /* the nodes list has been cleared - rebuild it in order */ while (NULL != (item = pmix_list_remove_first(&keep))) { pmix_list_append(nodes, item); } /* did they ask for more than we could provide */ if (!want_all_empty && 0 < num_empty) { pmix_show_help("help-dash-host.txt", "dash-host:not-enough-empty", true, num_empty); rc = PRTE_ERR_SILENT; goto cleanup; } rc = PRTE_SUCCESS; /* done filtering existing list */ cleanup: for (i = 0; i < len_mapped_node; i++) { if (NULL != mapped_nodes[i]) { free(mapped_nodes[i]); mapped_nodes[i] = NULL; } } if (NULL != mapped_nodes) { free(mapped_nodes); } return rc; } int prte_util_get_ordered_dash_host_list(pmix_list_t *nodes, char *hosts) { int rc, i; char **mapped_nodes = NULL; prte_node_t *node; if (PRTE_SUCCESS != (rc = parse_dash_host(&mapped_nodes, hosts))) { PRTE_ERROR_LOG(rc); } /* for each entry, create a node entry on the list */ for (i = 0; NULL != mapped_nodes[i]; i++) { node = PMIX_NEW(prte_node_t); node->name = strdup(mapped_nodes[i]); pmix_list_append(nodes, &node->super); } /* cleanup */ PMIx_Argv_free(mapped_nodes); return rc; } prrte-4.1.1~rc2/src/util/dash_host/dash_host.h0000664000175000017500000000273715203542132021515 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef PRTE_UTIL_DASH_HOST_H #define PRTE_UTIL_DASH_HOST_H #include "prte_config.h" #include "src/class/pmix_list.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS PRTE_EXPORT int prte_util_add_dash_host_nodes(pmix_list_t *nodes, char *hosts, bool allocating); PRTE_EXPORT int prte_util_filter_dash_host_nodes(pmix_list_t *nodes, char *hosts, bool remove); PRTE_EXPORT int prte_util_get_ordered_dash_host_list(pmix_list_t *nodes, char *hosts); PRTE_EXPORT int prte_util_dash_host_compute_slots(prte_node_t *node, char *hosts); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/dash_host/help-dash-host.txt0000664000175000017500000000776615203542132022760 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2025-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's dash-host. # [host] Host syntax consists of a comma-delimited list of node names, each entry optionally containing a ":N" extension indicating the number of slots to assign to that entry: --host node01:5,node02 In the absence of the slot extension, one slot will be assigned to the node. Duplicate entries are aggregated and the number of slots assigned to that node are summed together. Note: A "slot" is the PRRTE term for an allocatable unit where we can launch a process. Thus, the number of slots equates to the maximum number of processes PRRTE may start on that node without oversubscribing it. See the "Host specification" HTML documentation for details about the format and content of hostfiles. # [add-host] PRRTE allows a user to expand an existing DVM prior to launching an application. Users can specify a a comma-delimited list of node names, each entry optionally containing a ":N" extension indicating the number of slots to assign to that entry: --host node01:5,node02 In the absence of the slot extension, one slot will be assigned to the node. Duplicate entries are aggregated and the number of slots assigned to that node are summed together. Note: A "slot" is the PRRTE term for an allocatable unit where we can launch a process. Thus, the number of slots equates to the maximum number of processes PRRTE may start on that node without oversubscribing it. The list can include nodes that are already part of the DVM — in this case, the number of slots available on those nodes will be set to the new specification, or adjusted as directed: --host node01:5,node02 would direct that node01 be set to 5 slots and node02 will have 1 slot, while --host node01:+5,node02 would add 5 slots to the current value for node01, and --host node01:-5,node02 would subtract 5 slots from the current value. Slot adjustments for existing nodes will have no impact on currently executing jobs, but will be applied to any new spawn requests. Nodes contained in the add-host specification are available for immediate use by the accompanying application. Users desiring to constrain the accompanying application to the newly added nodes should also include the "--host" command line directive, giving the same hosts in its argument: --add-host node01:+5,node02 --host node01:5,node02 Note that the "--host" argument indicates the number of slots to assign node01 for this spawn request, and not the number of slots being added to the node01 allocation. # [not-all-mapped-alloc] At least one of the requested hosts is not included in the current allocation. Missing requested host: %s Please check your allocation or your request. # [dash-host:relative-node-not-found] A relative host was specified, but was not found. The requested host was specified with "--host" as: Index: %d Syntax given: %s Re-run this command with "--help hosts" for further information. # [dash-host:relative-node-out-of-bounds] A relative host was specified, but the index given is beyond the number of hosts in the current allocation: Index: %d #hosts: %d You could obtain a larger allocation or reduce the relative host index. Re-run this command with "--help hosts" for further information. # [dash-host:invalid-relative-node-syntax] A relative host was improperly specified — the value provided was. --host: %s You may have forgotten to preface a node with "N" or "n", or used the "e" or "E" to indicate empty nodes, or you ended the value with a colon but forgot to include the number of empty nodes you were requesting. Re-run this command with "--help hosts" for further information. # [dash-host:not-enough-empty] The requested number of empty hosts was not available — the system was short by %d hosts. Please recheck your allocation. Re-run this command with "--help hosts" for further information. prrte-4.1.1~rc2/src/util/error.h0000664000175000017500000000446515203542132016716 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2017 FUJITSU LIMITED. All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_UTIL_ERROR_H #define PRTE_UTIL_ERROR_H #include "prte_config.h" #include "src/util/pmix_output.h" BEGIN_C_DECLS #define PRTE_ERROR_LOG(r) \ do { \ if (PRTE_ERR_SILENT != (r)) { \ pmix_output(0, "PRTE ERROR: %s in file %s at line %d", prte_strerror((r)), __FILE__, \ __LINE__); \ } \ } while (0) /** * Return string for given error message * * Accepts an error number argument \c errnum and returns a pointer to * the corresponding message string. The result is returned in a * static buffer that should not be released with free(). * * If errnum is \c PRTE_ERR_IN_ERRNO, the system strerror is called * with an argument of the current value of \c errno and the resulting * string is returned. * * If the errnum is not a known value, the returned value may be * overwritten by subsequent calls to prte_strerror. */ PRTE_EXPORT const char *prte_strerror(int errnum); END_C_DECLS #endif /* PRTE_UTIL_ERROR_H */ prrte-4.1.1~rc2/src/util/name_fns.h0000664000175000017500000000767315203542132017357 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * Populates global structure with system-specific information. * * Notes: add limits.h, compute size of integer and other types via sizeof(type)*CHAR_BIT * */ #ifndef _PRTE_NAME_FNS_H_ #define _PRTE_NAME_FNS_H_ #include "prte_config.h" #ifdef HAVE_STDINT_h # include #endif #include "types.h" #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" BEGIN_C_DECLS typedef uint8_t prte_ns_cmp_bitmask_t; /**< Bit mask for comparing process names */ #define PRTE_NS_CMP_NONE 0x00 #define PRTE_NS_CMP_JOBID 0x02 #define PRTE_NS_CMP_VPID 0x04 #define PRTE_NS_CMP_ALL 0x0f #define PRTE_NS_CMP_WILD 0x10 /* useful define to print name args in output messages */ PRTE_EXPORT char *prte_util_print_name_args(const pmix_proc_t *name); #define PRTE_NAME_PRINT(n) prte_util_print_name_args(n) PRTE_EXPORT char *prte_util_print_jobids(const pmix_nspace_t job); #define PRTE_JOBID_PRINT(n) prte_util_print_jobids(n) PRTE_EXPORT char *prte_util_print_vpids(const pmix_rank_t vpid); #define PRTE_VPID_PRINT(n) prte_util_print_vpids(n) PRTE_EXPORT char *prte_util_print_job_family(const pmix_nspace_t job); #define PRTE_JOB_FAMILY_PRINT(n) prte_util_print_job_family(n) PRTE_EXPORT char *prte_util_print_local_jobid(const pmix_nspace_t job); #define PRTE_LOCAL_JOBID_PRINT(n) prte_util_print_local_jobid(n) PRTE_EXPORT char *prte_pretty_print_timing(int64_t secs, int64_t usecs); #define PRTE_LOCAL_JOBID(n) prte_util_get_local_jobid(n) __prte_attribute_always_inline__ static inline int prte_util_get_local_jobid(pmix_nspace_t n) { char *ptr; int lid; ptr = strrchr(n, '@'); if (NULL == ptr) { /* this isn't a PRRTE job */ return -1; } ++ptr; lid = strtoul(ptr, NULL, 10); return lid; } /* List of names for general use */ struct prte_namelist_t { pmix_list_item_t super; /**< Allows this item to be placed on a list */ pmix_proc_t name; /**< Name of a process */ }; typedef struct prte_namelist_t prte_namelist_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_namelist_t); PRTE_EXPORT int prte_util_convert_vpid_to_string(char **vpid_string, const pmix_rank_t vpid); PRTE_EXPORT int prte_util_convert_string_to_process_name(pmix_proc_t *name, const char *name_string); PRTE_EXPORT int prte_util_convert_process_name_to_string(char **name_string, const pmix_proc_t *name); PRTE_EXPORT int prte_util_compare_name_fields(prte_ns_cmp_bitmask_t fields, const pmix_proc_t *name1, const pmix_proc_t *name2); PRTE_EXPORT char *prte_util_make_version_string(const char *scope, int major, int minor, int release, const char *greek, const char *repo); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/proc_info.h0000664000175000017500000001167015203542132017537 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * Populates global structure with process-specific information. * * */ #ifndef _PRTE_PROC_INFO_H_ #define _PRTE_PROC_INFO_H_ #include "prte_config.h" #include #ifdef HAVE_SYS_TYPES_H # include #endif #include "types.h" #include "src/hwloc/hwloc-internal.h" #include BEGIN_C_DECLS typedef uint8_t prte_proc_type_t; #define PRTE_PROC_TYPE_NONE 0x0000 #define PRTE_PROC_DAEMON 0x0002 #define PRTE_PROC_MASTER 0x0004 #define PRTE_PROC_IS_DAEMON (PRTE_PROC_DAEMON & prte_process_info.proc_type) #define PRTE_PROC_IS_MASTER (PRTE_PROC_MASTER & prte_process_info.proc_type) /** * Process information structure * * The prte_proc_info() function fills the pid field and obtains the * process name, storing that information in the global structure. The * structure also holds path names to the universe, job, and process * session directories, and to the stdin, stdout, and stderr temp * files - however, these are all initialized elsewhere. */ typedef struct prte_process_info_t { pmix_proc_t myproc; pmix_proc_t my_hnp; /**< Name of my hnp */ char *my_hnp_uri; /**< Contact info for my hnp */ pmix_proc_t my_parent; /**< Name of my parent (or my HNP if no parent was specified) */ pid_t hnp_pid; /**< hnp pid - used if singleton */ pmix_rank_t num_daemons; /**< number of daemons in system */ int num_nodes; /**< number of nodes in the job */ char *nodename; /**< string name for this node */ char **aliases; /**< aliases for this node */ uid_t uid; /**< Real user UD */ uid_t euid; /**< Effective user ID */ gid_t gid; /**< Real group ID */ gid_t egid; /**< Effective group ID */ pid_t pid; /**< Local process ID for this process */ prte_proc_type_t proc_type; /**< Type of process */ char *my_uri; /**< My contact info */ uint16_t my_port; /**< TCP port for out-of-band comm */ /* The session directory has the form * ///, where the prefix * can either be provided by the user via the * --tmpdir command-line flag, the use of one of several * environmental variables, or else a default location. */ char *tmpdir_base; /**< Base directory of the session dir tree */ char *sessdir_prefix; /**< starting string for top session dir name */ char *top_session_dir; /**< Top-most directory of the session tree */ bool rm_session_dirs; /**< Session directories will be cleaned up by RM */ char *cpuset; /**< String-representation of bitmap where we are bound */ bool shared_fs; // whether the tmpdir is on a shared file system } prte_process_info_t; /** * * Global process info descriptor. Initialized to almost no * meaningful information - data is provided by calling \c * prte_rte_init() (which calls \c prte_proc_info() to fill in the * structure). * * The exception to this rule is the \c prte_process_info.seed field, * which will be initialized to \c false, but should be set to \c true * before calling \c prte_rte_info() if the caller is a seed daemon. */ PRTE_EXPORT extern prte_process_info_t prte_process_info; /** * \internal * * Global structure to store a wide range of information about the * process. prte_proc_info populates a global variable with * information about the process being executing. This function should * be called only once, from prte_rte_init(). * * @param None. * * @retval PRTE_SUCCESS Successfully initialized the various fields. * @retval OMPI_ERROR Failed to initialize one or more fields. */ PRTE_EXPORT int prte_proc_info(void); PRTE_EXPORT int prte_proc_info_finalize(void); PRTE_EXPORT void prte_setup_hostname(void); PRTE_EXPORT bool prte_check_host_is_local(const char *name); END_C_DECLS #endif prrte-4.1.1~rc2/src/util/Makefile.am0000664000175000017500000000632115203542132017441 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2013 NVIDIA Corporation. All rights reserved. # Copyright (c) 2013-2020 Intel, Inc. All rights reserved. # Copyright (c) 2016 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2016-2017 IBM Corporation. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # Copyright (c) 2023 Jeffrey M. Squyres. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # SUBDIRS = hostfile noinst_LTLIBRARIES = libprrteutil.la AM_CPPFLAGS = $(LTDLINCL) include $(top_srcdir)/Makefile.prte-rules EXTRA_DIST = \ dash_host/help-dash-host.txt \ hostfile/help-hostfile.txt \ help-prte-util.txt \ prte-convert-help.py # Source code files headers = \ attr.h \ bit_ops.h \ prte_cmd_line.h \ crc.h \ daemon_init.h \ dash_host/dash_host.h \ error_strings.h \ ethtool.h \ error.h \ malloc.h \ name_fns.h \ nidmap.h \ numtostr.h \ proc_info.h \ session_dir.h \ stacktrace.h \ sys_limits.h \ uri.h sources = \ attr.c \ crc.c \ daemon_init.c \ dash_host/dash_host.c \ error_strings.c \ ethtool.c \ error.c \ malloc.c \ name_fns.c \ nidmap.c \ numtostr.c \ proc_info.c \ session_dir.c \ stacktrace.c \ sys_limits.c \ uri.c \ prte_show_help_content.c libprrteutil_la_SOURCES = $(headers) $(sources) libprrteutil_la_LIBADD = \ hostfile/libprrteutilhostfile.la libprrteutil_la_DEPENDENCIES = \ hostfile/libprrteutilhostfile.la prte_show_help_content.c: prte-convert-help.py if PRTE_PICKY_COMPILERS $(PRTE_V_GEN) $(PYTHON) $(abs_srcdir)/prte-convert-help.py \ --root $(abs_top_srcdir) \ --out prte_show_help_content.c \ --purge else $(PRTE_V_GEN) $(PYTHON) $(abs_srcdir)/prte-convert-help.py \ --root $(abs_top_srcdir) \ --out prte_show_help_content.c endif # Conditionally install the header files if WANT_INSTALL_HEADERS prtedir = $(prteincludedir)/$(subdir) prte_HEADERS = $(headers) endif clean-local: rm -f prte_show_help_content.c prte_show_help_content.lo MAINTAINERCLEANFILES = \ prte_show_help_content.c distclean-local: rm -f prte_show_help_content.c prrte-4.1.1~rc2/src/util/daemon_init.h0000664000175000017500000000413015203542132020040 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #ifndef PRTE_DAEMON_INIT_H #define PRTE_DAEMON_INIT_H #include "prte_config.h" BEGIN_C_DECLS /* * Turn a process into a daemon. * * This function converts a process into a daemon in an orderly manner. It first forks a child * process, then the parent exits. The child continues on to become a session leader, reset the file * mode creation mask, and changes working directories to the one specified. * * @param working_dir Pointer to a character string containing the desired working directory. * Providing a value of NULL will cause the function to leave the program in the current working * directory. * @param parent_fn The function to execute in the parent before exiting * a value of NULL will cause the parent to simply exit(0). * * @retval PRTE_SUCCESS Indicates that the conversion was successful * @retval PRTE_ERROR Indicates that the conversion was not successful - a fork could not be * completed. */ PRTE_EXPORT int prte_daemon_init_callback(char *working_dir, int (*parent_fn)(pid_t child)); END_C_DECLS static inline int prte_daemon_init(char *working_dir) { return prte_daemon_init_callback(working_dir, NULL); } END_C_DECLS #endif /* PRTE_DAEMON_INIT_H */ prrte-4.1.1~rc2/src/util/uri.h0000664000175000017500000000622015203542132016353 0ustar alastairalastair/* * Copyright (c) 2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * @file */ /* Per RFC3986: The generic URI syntax consists of a hierarchical sequence of components referred to as the scheme, authority, path, query, and fragment. URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] hier-part = "//" authority path-abempty / path-absolute / path-rootless / path-empty The scheme and path components are required, though the path may be empty (no characters). When authority is present, the path must either be empty or begin with a slash ("/") character. When authority is not present, the path cannot begin with two slash characters ("//"). These restrictions result in five different ABNF rules for a path (Section 3.3), only one of which will match any given URI reference. The following are two example URIs and their component parts: foo://example.com:8042/over/there?name=ferret#nose \_/ \______________/\_________/ \_________/ \__/ | | | | | scheme authority path query fragment | _____________________|__ / \ / \ urn:example:animal:ferret:nose Initially, this code supports only part of the first example - a scheme followed by an authority and a path. Queries and fragments are not supported. The APIs are modeled on the Gnome equivalent functions, though the code is NOT in any way based on thoat code. */ #ifndef PRTE_URI_H #define PRTE_URI_H #include "prte_config.h" #ifdef HAVE_UNISTD_H # include #endif BEGIN_C_DECLS /** * Parse a uri to retrieve the scheme * * The caller is responsible for freeing the returned string. */ PRTE_EXPORT char *prte_uri_get_scheme(const char *uri) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; /** * Create a uri from a hostname and filename * * The caller is responsible for freeing the returned string. */ PRTE_EXPORT char *prte_filename_to_uri(const char *filename, const char *hostname) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; /** * Extract the filename (and hostname) from a uri * * @param uri : a uri describing a filename (escaped, encoded in ASCII). * @param hostname : Location to store hostname for the URI, or NULL. * If there is no hostname in the URI, NULL will be * stored in this location. * @retval a newly-allocated string holding the resulting filename, or NULL on an error. * * The caller is responsible for freeing the returned string. */ PRTE_EXPORT char *prte_filename_from_uri(const char *uri, char **hostname) __prte_attribute_malloc__ __prte_attribute_warn_unused_result__; END_C_DECLS #endif /* PRTE_URI_H */ prrte-4.1.1~rc2/src/util/crc.h0000664000175000017500000001223315203542132016324 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009 IBM Corporation. All rights reserved. * Copyright (c) 2009 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef _PRTE_CRC_H_ #define _PRTE_CRC_H_ #include "prte_config.h" #include BEGIN_C_DECLS #define CRC_POLYNOMIAL ((unsigned int) 0x04c11db7) #define CRC_INITIAL_REGISTER ((unsigned int) 0xffffffff) #define PRTE_CSUM(SRC, LEN) prte_uicsum(SRC, LEN) #define PRTE_CSUM_PARTIAL(SRC, LEN, UI1, UI2) prte_uicsum_partial(SRC, LEN, UI1, UI2) #define PRTE_CSUM_BCOPY_PARTIAL(SRC, DST, LEN1, LEN2, UI1, UI2) \ prte_bcopy_uicsum_partial(SRC, DST, LEN1, LEN2, UI1, UI2) #define PRTE_CSUM_ZERO 0 PRTE_EXPORT unsigned long prte_bcopy_csum_partial(const void *source, void *destination, size_t copylen, size_t csumlen, unsigned long *lastPartialLong, size_t *lastPartialLength); static inline unsigned long prte_bcopy_csum(const void *source, void *destination, size_t copylen, size_t csumlen) { unsigned long plong = 0; size_t plength = 0; return prte_bcopy_csum_partial(source, destination, copylen, csumlen, &plong, &plength); } PRTE_EXPORT unsigned int prte_bcopy_uicsum_partial(const void *source, void *destination, size_t copylen, size_t csumlen, unsigned int *lastPartialInt, size_t *lastPartialLength); static inline unsigned int prte_bcopy_uicsum(const void *source, void *destination, size_t copylen, size_t csumlen) { unsigned int pint = 0; size_t plength = 0; return prte_bcopy_uicsum_partial(source, destination, copylen, csumlen, &pint, &plength); } PRTE_EXPORT unsigned long prte_csum_partial(const void *source, size_t csumlen, unsigned long *lastPartialLong, size_t *lastPartialLength); static inline unsigned long prte_csum(const void *source, size_t csumlen) { unsigned long lastPartialLong = 0; size_t lastPartialLength = 0; return prte_csum_partial(source, csumlen, &lastPartialLong, &lastPartialLength); } /* * The buffer passed to this function is assumed to be 16-bit aligned */ static inline uint16_t prte_csum16(const void *source, size_t csumlen) { uint16_t *src = (uint16_t *) source; register uint32_t csum = 0; while (csumlen > 1) { csum += *src++; csumlen -= 2; } /* Add leftover byte, if any */ if (csumlen > 0) csum += *((unsigned char *) src); /* Fold 32-bit checksum to 16 bits */ while (csum >> 16) { csum = (csum & 0xFFFF) + (csum >> 16); } return csum; } PRTE_EXPORT unsigned int prte_uicsum_partial(const void *source, size_t csumlen, unsigned int *lastPartialInt, size_t *lastPartialLength); static inline unsigned int prte_uicsum(const void *source, size_t csumlen) { unsigned int lastPartialInt = 0; size_t lastPartialLength = 0; return prte_uicsum_partial(source, csumlen, &lastPartialInt, &lastPartialLength); } /* * CRC Support */ void prte_initialize_crc_table(void); PRTE_EXPORT unsigned int prte_bcopy_uicrc_partial(const void *source, void *destination, size_t copylen, size_t crclen, unsigned int partial_crc); static inline unsigned int prte_bcopy_uicrc(const void *source, void *destination, size_t copylen, size_t crclen) { return prte_bcopy_uicrc_partial(source, destination, copylen, crclen, CRC_INITIAL_REGISTER); } PRTE_EXPORT unsigned int prte_uicrc_partial(const void *source, size_t crclen, unsigned int partial_crc); static inline unsigned int prte_uicrc(const void *source, size_t crclen) { return prte_uicrc_partial(source, crclen, CRC_INITIAL_REGISTER); } END_C_DECLS #endif prrte-4.1.1~rc2/src/util/error.c0000664000175000017500000003320515203542132016703 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 FUJITSU LIMITED. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #include #include #include #include "constants.h" #include "src/runtime/prte_globals.h" #include "src/util/error.h" #include "src/util/pmix_printf.h" #include "src/util/proc_info.h" #include "src/util/pmix_string_copy.h" const char *prte_strerror(int errnum) { const char *retval; switch (errnum) { case PRTE_SUCCESS: retval = "Success"; break; case PRTE_ERROR: retval = "Error"; break; case PRTE_ERR_OUT_OF_RESOURCE: retval = "Out of resource"; break; case PRTE_ERR_TEMP_OUT_OF_RESOURCE: retval = "Temporarily out of resource"; break; case PRTE_ERR_RESOURCE_BUSY: retval = "Resource busy"; break; case PRTE_ERR_BAD_PARAM: retval = "Bad parameter"; break; case PRTE_ERR_FATAL: retval = "Fatal"; break; case PRTE_ERR_NOT_IMPLEMENTED: retval = "Not implemented"; break; case PRTE_ERR_NOT_SUPPORTED: retval = "Not supported"; break; case PRTE_ERR_INTERRUPTED: retval = "Interrupted"; break; case PRTE_ERR_WOULD_BLOCK: retval = "Would block"; break; case PRTE_ERR_IN_ERRNO: retval = "In errno"; break; case PRTE_ERR_UNREACH: retval = "Unreachable"; break; case PRTE_ERR_NOT_FOUND: retval = "Not found"; break; case PRTE_EXISTS: retval = "Exists"; break; case PRTE_ERR_TIMEOUT: retval = "Timeout"; break; case PRTE_ERR_NOT_AVAILABLE: retval = "Not available"; break; case PRTE_ERR_PERM: retval = "No permission"; break; case PRTE_ERR_VALUE_OUT_OF_BOUNDS: retval = "Value out of bounds"; break; case PRTE_ERR_FILE_READ_FAILURE: retval = "File read failure"; break; case PRTE_ERR_FILE_WRITE_FAILURE: retval = "File write failure"; break; case PRTE_ERR_FILE_OPEN_FAILURE: retval = "File open failure"; break; case PRTE_ERR_PACK_MISMATCH: retval = "Pack data mismatch"; break; case PRTE_ERR_PACK_FAILURE: retval = "Data pack failed"; break; case PRTE_ERR_UNPACK_FAILURE: retval = "Data unpack failed"; break; case PRTE_ERR_UNPACK_INADEQUATE_SPACE: retval = "Data unpack had inadequate space"; break; case PRTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER: retval = "Data unpack would read past end of buffer"; break; case PRTE_ERR_OPERATION_UNSUPPORTED: retval = "Requested operation is not supported on referenced data type"; break; case PRTE_ERR_UNKNOWN_DATA_TYPE: retval = "Unknown data type"; break; case PRTE_ERR_BUFFER: retval = "Buffer type (described vs non-described) mismatch - operation not allowed"; break; case PRTE_ERR_DATA_TYPE_REDEF: retval = "Attempt to redefine an existing data type"; break; case PRTE_ERR_DATA_OVERWRITE_ATTEMPT: retval = "Attempt to overwrite a data value"; break; case PRTE_ERR_MODULE_NOT_FOUND: retval = "Framework requires at least one active module, but none found"; break; case PRTE_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED: retval = "OS topology does not support slot_list process affinity"; break; case PRTE_ERR_TOPO_SOCKET_NOT_SUPPORTED: retval = "Could not obtain socket topology information"; break; case PRTE_ERR_TOPO_CORE_NOT_SUPPORTED: retval = "Could not obtain core topology information"; break; case PRTE_ERR_NOT_ENOUGH_SOCKETS: retval = "Not enough sockets to meet request"; break; case PRTE_ERR_NOT_ENOUGH_CORES: retval = "Not enough cores to meet request"; break; case PRTE_ERR_INVALID_PHYS_CPU: retval = "Invalid physical cpu number returned"; break; case PRTE_ERR_MULTIPLE_AFFINITIES: retval = "Multiple methods for assigning process affinity were specified"; break; case PRTE_ERR_SLOT_LIST_RANGE: retval = "Provided slot_list range is invalid"; break; case PRTE_ERR_NETWORK_NOT_PARSEABLE: retval = "Provided network specification is not parseable"; break; case PRTE_ERR_NOT_INITIALIZED: retval = "Not initialized"; break; case PRTE_ERR_NOT_BOUND: retval = "Not bound"; break; case PRTE_ERR_PROC_ENTRY_NOT_FOUND: retval = "Database entry not found"; break; case PRTE_ERR_DATA_VALUE_NOT_FOUND: retval = "Data for specified key not found"; break; case PRTE_ERR_CONNECTION_FAILED: retval = "Connection failed"; break; case PRTE_ERR_AUTHENTICATION_FAILED: retval = "Authentication failed"; break; case PRTE_ERR_COMM_FAILURE: retval = "Comm failure"; break; case PRTE_ERR_SERVER_NOT_AVAIL: retval = "Server not available"; break; case PRTE_ERR_IN_PROCESS: retval = "Operation in process"; break; case PRTE_ERR_DEBUGGER_RELEASE: retval = "Release debugger"; break; case PRTE_ERR_HANDLERS_COMPLETE: retval = "Event handlers complete"; break; case PRTE_ERR_PARTIAL_SUCCESS: retval = "Partial success"; break; case PRTE_ERR_PROC_ABORTED: retval = "Process abnormally terminated"; break; case PRTE_ERR_PROC_REQUESTED_ABORT: retval = "Process requested abort"; break; case PRTE_ERR_PROC_ABORTING: retval = "Process is aborting"; break; case PRTE_ERR_NODE_DOWN: retval = "Node has gone down"; break; case PRTE_ERR_NODE_OFFLINE: retval = "Node has gone offline"; break; case PRTE_ERR_JOB_TERMINATED: retval = "Job terminated"; break; case PRTE_ERR_PROC_RESTART: retval = "Process restarted"; break; case PRTE_ERR_PROC_CHECKPOINT: retval = "Process checkpoint"; break; case PRTE_ERR_PROC_MIGRATE: retval = "Process migrate"; break; case PRTE_ERR_EVENT_REGISTRATION: retval = "Event registration"; break; case PRTE_ERR_HEARTBEAT_ALERT: retval = "Heartbeat not received"; break; case PRTE_ERR_FILE_ALERT: retval = "File alert - proc may have stalled"; break; case PRTE_ERR_RECV_LESS_THAN_POSTED: retval = "Receive was less than posted size"; break; case PRTE_ERR_RECV_MORE_THAN_POSTED: retval = "Receive was greater than posted size"; break; case PRTE_ERR_NO_MATCH_YET: retval = "No match for receive posted"; break; case PRTE_ERR_REQUEST: retval = "Request error"; break; case PRTE_ERR_NO_CONNECTION_ALLOWED: retval = "No connection allowed"; break; case PRTE_ERR_CONNECTION_REFUSED: retval = "Connection refused"; break; case PRTE_ERR_TYPE_MISMATCH: retval = "Type mismatch"; break; case PRTE_ERR_COMPARE_FAILURE: retval = "Data comparison failure"; break; case PRTE_ERR_COPY_FAILURE: retval = "Data copy failure"; break; case PRTE_ERR_PROC_STATE_MISSING: retval = "The process state information is missing on the registry"; break; case PRTE_ERR_PROC_EXIT_STATUS_MISSING: retval = "The process exit status is missing on the registry"; break; case PRTE_ERR_INDETERMINATE_STATE_INFO: retval = "Request for state returned multiple responses"; break; case PRTE_ERR_NODE_FULLY_USED: retval = "All the slots on a given node have been used"; break; case PRTE_ERR_INVALID_NUM_PROCS: retval = "Multiple applications were specified, but at least one failed to specify the " "number of processes to run"; break; case PRTE_ERR_SILENT: if (prte_report_silent_errors) { retval = "Silent error"; } else { retval = ""; } break; case PRTE_ERR_ADDRESSEE_UNKNOWN: retval = "A message is attempting to be sent to a process whose contact information is " "unknown"; break; case PRTE_ERR_SYS_LIMITS_PIPES: retval = "The system limit on number of pipes a process can open was reached"; break; case PRTE_ERR_PIPE_SETUP_FAILURE: retval = "A pipe could not be setup between a daemon and one of its local processes"; break; case PRTE_ERR_SYS_LIMITS_CHILDREN: retval = "The system limit on number of children a process can have was reached"; break; case PRTE_ERR_FAILED_GET_TERM_ATTRS: retval = "The I/O forwarding system was unable to get the attributes of your terminal"; break; case PRTE_ERR_WDIR_NOT_FOUND: retval = "The specified working directory could not be found"; break; case PRTE_ERR_EXE_NOT_FOUND: retval = "The specified executable could not be found"; break; case PRTE_ERR_PIPE_READ_FAILURE: retval = "A pipe could not be read"; break; case PRTE_ERR_EXE_NOT_ACCESSIBLE: retval = "The specified executable could not be executed"; break; case PRTE_ERR_FAILED_TO_START: retval = "The specified application failed to start"; break; case PRTE_ERR_FILE_NOT_EXECUTABLE: retval = "A system-required executable either could not be found or was not executable by " "this user"; break; case PRTE_ERR_HNP_COULD_NOT_START: retval = "Unable to start a daemon on the local node"; break; case PRTE_ERR_SYS_LIMITS_SOCKETS: retval = "The system limit on number of network connections a process can open was reached"; break; case PRTE_ERR_SOCKET_NOT_AVAILABLE: retval = "Unable to open a TCP socket for out-of-band communications"; break; case PRTE_ERR_SYSTEM_WILL_BOOTSTRAP: retval = "System will determine resources during bootstrap of daemons"; break; case PRTE_ERR_RESTART_LIMIT_EXCEEDED: retval = "Limit on number of process restarts was exceeded"; break; case PRTE_ERR_INVALID_NODE_RANK: retval = "Invalid node rank"; break; case PRTE_ERR_INVALID_LOCAL_RANK: retval = "Invalid local rank"; break; case PRTE_ERR_UNRECOVERABLE: retval = "Unrecoverable error"; break; case PRTE_ERR_MEM_LIMIT_EXCEEDED: retval = "Memory limit exceeded"; break; case PRTE_ERR_HEARTBEAT_LOST: retval = "Heartbeat lost"; break; case PRTE_ERR_PROC_STALLED: retval = "Proc appears to be stalled"; break; case PRTE_ERR_NO_APP_SPECIFIED: retval = "No application specified"; break; case PRTE_ERR_NO_EXE_SPECIFIED: retval = "No executable specified"; break; case PRTE_ERR_COMM_DISABLED: retval = "Communications have been disabled"; break; case PRTE_ERR_FAILED_TO_MAP: retval = "Unable to map job"; break; case PRTE_ERR_TAKE_NEXT_OPTION: if (prte_report_silent_errors) { retval = "Next option"; } else { retval = ""; } break; case PRTE_ERR_SENSOR_LIMIT_EXCEEDED: retval = "Sensor limit exceeded"; break; case PRTE_ERR_ALLOCATION_PENDING: retval = "Allocation pending"; break; case PRTE_ERR_NO_PATH_TO_TARGET: retval = "No OOB path to target"; break; case PRTE_ERR_OP_IN_PROGRESS: retval = "Operation in progress"; break; case PRTE_ERR_OPEN_CONDUIT_FAIL: retval = "Open messaging conduit failed"; break; case PRTE_ERR_OUT_OF_ORDER_MSG: retval = "Out of order message"; break; case PRTE_ERR_FORCE_SELECT: retval = "Force select"; break; case PRTE_ERR_JOB_CANCELLED: retval = "Job cancelled"; break; case PRTE_ERR_CONDUIT_SEND_FAIL: retval = " Transport Conduit returned send error"; break; default: retval = "Unknown error"; } return retval; } prrte-4.1.1~rc2/src/util/ethtool.h0000664000175000017500000000104215203542132017227 0ustar alastairalastair/* * Copyright (c) 2016 Karol Mroz. All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ETHTOOL_H #define PRTE_ETHTOOL_H /* * Obtain an appropriate bandwidth for the interface if_name. On Linux, we * get this via an ioctl(). Elsewhere or in the error case, we return the * speed as 0. */ unsigned int prte_ethtool_get_speed(const char *if_name); #endif prrte-4.1.1~rc2/src/util/sys_limits.h0000664000175000017500000000350115203542132017752 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_SYS_LIMITS_H #define PRTE_SYS_LIMITS_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif BEGIN_C_DECLS /* define a structure to hold the various limits we find * so that users can neatly access them */ typedef struct prte_sys_limits_t { bool initialized; int num_files; int num_procs; size_t file_size; } prte_sys_limits_t; /* since we only want to do this once, we will store the * values in the following locations - provide access here */ PRTE_EXPORT extern prte_sys_limits_t prte_sys_limits; /* Get the system resource limits and, if requested, set * them to the specified limit */ PRTE_EXPORT int prte_util_init_sys_limits(char **errmsg); /** * Get pagesize */ PRTE_EXPORT int prte_getpagesize(void); END_C_DECLS #endif /* PRTE_STRNCPY_H */ prrte-4.1.1~rc2/src/util/error_strings.c0000664000175000017500000002106715203542132020457 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_SYS_SIGNAL_H # include #else # include #endif #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" #include "src/util/error_strings.h" const char *prte_job_state_to_str(prte_job_state_t state) { switch (state) { case PRTE_JOB_STATE_UNDEF: return "UNDEFINED"; case PRTE_JOB_STATE_INIT: return "PENDING INIT"; case PRTE_JOB_STATE_INIT_COMPLETE: return "INIT_COMPLETE"; case PRTE_JOB_STATE_ALLOCATE: return "PENDING ALLOCATION"; case PRTE_JOB_STATE_ALLOCATION_COMPLETE: return "ALLOCATION COMPLETE"; case PRTE_JOB_STATE_MAP: return "PENDING MAPPING"; case PRTE_JOB_STATE_MAP_COMPLETE: return "MAP COMPLETE"; case PRTE_JOB_STATE_SYSTEM_PREP: return "PENDING FINAL SYSTEM PREP"; case PRTE_JOB_STATE_LAUNCH_DAEMONS: return "PENDING DAEMON LAUNCH"; case PRTE_JOB_STATE_DAEMONS_LAUNCHED: return "DAEMONS LAUNCHED"; case PRTE_JOB_STATE_DAEMONS_REPORTED: return "ALL DAEMONS REPORTED"; case PRTE_JOB_STATE_VM_READY: return "VM READY"; case PRTE_JOB_STATE_LAUNCH_APPS: return "PENDING APP LAUNCH"; case PRTE_JOB_STATE_SEND_LAUNCH_MSG: return "SENDING LAUNCH MSG"; case PRTE_JOB_STATE_RUNNING: return "RUNNING"; case PRTE_JOB_STATE_SUSPENDED: return "SUSPENDED"; case PRTE_JOB_STATE_REGISTERED: return "SYNC REGISTERED"; case PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE: return "LOCAL LAUNCH COMPLETE"; case PRTE_JOB_STATE_READY_FOR_DEBUG: return "READY FOR DEBUG"; case PRTE_JOB_STATE_STARTED: return "JOB STARTED"; case PRTE_JOB_STATE_UNTERMINATED: return "UNTERMINATED"; case PRTE_JOB_STATE_TERMINATED: return "NORMALLY TERMINATED"; case PRTE_JOB_STATE_NOTIFY_COMPLETED: return "NOTIFY COMPLETED"; case PRTE_JOB_STATE_NOTIFIED: return "NOTIFIED"; case PRTE_JOB_STATE_ALL_JOBS_COMPLETE: return "ALL JOBS COMPLETE"; case PRTE_JOB_STATE_ERROR: return "ARTIFICIAL BOUNDARY - ERROR"; case PRTE_JOB_STATE_KILLED_BY_CMD: return "KILLED BY INTERNAL COMMAND"; case PRTE_JOB_STATE_ABORTED: return "ABORTED"; case PRTE_JOB_STATE_FAILED_TO_START: return "FAILED TO START"; case PRTE_JOB_STATE_ABORTED_BY_SIG: return "ABORTED BY SIGNAL"; case PRTE_JOB_STATE_ABORTED_WO_SYNC: return "TERMINATED WITHOUT SYNC"; case PRTE_JOB_STATE_COMM_FAILED: return "COMMUNICATION FAILURE"; case PRTE_JOB_STATE_SENSOR_BOUND_EXCEEDED: return "SENSOR BOUND EXCEEDED"; case PRTE_JOB_STATE_CALLED_ABORT: return "PROC CALLED ABORT"; case PRTE_JOB_STATE_HEARTBEAT_FAILED: return "HEARTBEAT FAILED"; case PRTE_JOB_STATE_NEVER_LAUNCHED: return "NEVER LAUNCHED"; case PRTE_JOB_STATE_ABORT_ORDERED: return "ABORT IN PROGRESS"; case PRTE_JOB_STATE_NON_ZERO_TERM: return "AT LEAST ONE PROCESS EXITED WITH NON-ZERO STATUS"; case PRTE_JOB_STATE_FAILED_TO_LAUNCH: return "FAILED TO LAUNCH"; case PRTE_JOB_STATE_FORCED_EXIT: return "FORCED EXIT"; case PRTE_JOB_STATE_DAEMONS_TERMINATED: return "DAEMONS TERMINATED"; case PRTE_JOB_STATE_SILENT_ABORT: return "ERROR REPORTED ELSEWHERE"; case PRTE_JOB_STATE_REPORT_PROGRESS: return "REPORT PROGRESS"; case PRTE_JOB_STATE_ALLOC_FAILED: return "ALLOCATION FAILED"; case PRTE_JOB_STATE_MAP_FAILED: return "MAP FAILED"; case PRTE_JOB_STATE_CANNOT_LAUNCH: return "CANNOT LAUNCH"; case PRTE_JOB_STATE_FILES_POSN_FAILED: return "FILE PREPOSITION FAILED"; case PRTE_JOB_STATE_FT_CHECKPOINT: return "FAULT TOLERANCE CHECKPOINT"; case PRTE_JOB_STATE_FT_CONTINUE: return "FAULT TOLERANCE CONTINUE"; case PRTE_JOB_STATE_FT_RESTART: return "FAULT TOLERANCE RESTART"; case PRTE_JOB_STATE_ANY: return "ANY"; default: return "UNKNOWN STATE!"; } } const char *prte_app_ctx_state_to_str(prte_app_state_t state) { switch (state) { case PRTE_APP_STATE_UNDEF: return "UNDEFINED"; case PRTE_APP_STATE_INIT: return "PENDING INIT"; case PRTE_APP_STATE_ALL_MAPPED: return "ALL MAPPED"; case PRTE_APP_STATE_RUNNING: return "RUNNING"; case PRTE_APP_STATE_COMPLETED: return "COMPLETED"; default: return "UNKNOWN STATE!"; } } const char *prte_proc_state_to_str(prte_proc_state_t state) { switch (state) { case PRTE_PROC_STATE_UNDEF: return "UNDEFINED"; case PRTE_PROC_STATE_INIT: return "INITIALIZED"; case PRTE_PROC_STATE_RESTART: return "RESTARTING"; case PRTE_PROC_STATE_TERMINATE: return "MARKED FOR TERMINATION"; case PRTE_PROC_STATE_RUNNING: return "RUNNING"; case PRTE_PROC_STATE_REGISTERED: return "SYNC REGISTERED"; case PRTE_PROC_STATE_IOF_COMPLETE: return "IOF COMPLETE"; case PRTE_PROC_STATE_WAITPID_FIRED: return "WAITPID FIRED"; case PRTE_PROC_STATE_MODEX_READY: return "MODEX READY"; case PRTE_PROC_STATE_READY_FOR_DEBUG: return "READY FOR DEBUG"; case PRTE_PROC_STATE_UNTERMINATED: return "UNTERMINATED"; case PRTE_PROC_STATE_TERMINATED: return "NORMALLY TERMINATED"; case PRTE_PROC_STATE_ERROR: return "ARTIFICIAL BOUNDARY - ERROR"; case PRTE_PROC_STATE_KILLED_BY_CMD: return "KILLED BY INTERNAL COMMAND"; case PRTE_PROC_STATE_ABORTED: return "ABORTED"; case PRTE_PROC_STATE_FAILED_TO_START: return "FAILED TO START"; case PRTE_PROC_STATE_ABORTED_BY_SIG: return "ABORTED BY SIGNAL"; case PRTE_PROC_STATE_TERM_WO_SYNC: return "TERMINATED WITHOUT SYNC"; case PRTE_PROC_STATE_COMM_FAILED: return "COMMUNICATION FAILURE"; case PRTE_PROC_STATE_SENSOR_BOUND_EXCEEDED: return "SENSOR BOUND EXCEEDED"; case PRTE_PROC_STATE_CALLED_ABORT: return "CALLED ABORT"; case PRTE_PROC_STATE_HEARTBEAT_FAILED: return "HEARTBEAT FAILED"; case PRTE_PROC_STATE_MIGRATING: return "MIGRATING"; case PRTE_PROC_STATE_CANNOT_RESTART: return "CANNOT BE RESTARTED"; case PRTE_PROC_STATE_TERM_NON_ZERO: return "EXITED WITH NON-ZERO STATUS"; case PRTE_PROC_STATE_FAILED_TO_LAUNCH: return "FAILED TO LAUNCH"; case PRTE_PROC_STATE_UNABLE_TO_SEND_MSG: return "UNABLE TO SEND MSG"; case PRTE_PROC_STATE_LIFELINE_LOST: return "LIFELINE LOST"; case PRTE_PROC_STATE_NO_PATH_TO_TARGET: return "NO PATH TO TARGET"; case PRTE_PROC_STATE_FAILED_TO_CONNECT: return "FAILED TO CONNECT"; case PRTE_PROC_STATE_PEER_UNKNOWN: return "PEER UNKNOWN"; case PRTE_PROC_STATE_ANY: return "ANY"; default: return "UNKNOWN STATE!"; } } const char *prte_node_state_to_str(prte_node_state_t state) { switch (state) { case PRTE_NODE_STATE_UNDEF: return "UNDEF"; case PRTE_NODE_STATE_UNKNOWN: return "UNKNOWN"; case PRTE_NODE_STATE_DOWN: return "DOWN"; case PRTE_NODE_STATE_UP: return "UP"; case PRTE_NODE_STATE_REBOOT: return "REBOOT"; case PRTE_NODE_STATE_DO_NOT_USE: return "DO_NOT_USE"; case PRTE_NODE_STATE_NOT_INCLUDED: return "NOT_INCLUDED"; case PRTE_NODE_STATE_ADDED: return "ADDED"; default: return "UNKNOWN STATE!"; } } prrte-4.1.1~rc2/src/util/name_fns.c0000664000175000017500000003434115203542132017342 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include #include "src/threads/pmix_tsd.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/util/name_fns.h" #define PRTE_PRINT_NAME_ARGS_MAX_SIZE 1024 #define PRTE_PRINT_NAME_ARG_NUM_BUFS 16 /* constructor - used to initialize namelist instance */ static void prte_namelist_construct(prte_namelist_t *list) { PMIX_LOAD_PROCID(&list->name, NULL, PMIX_RANK_INVALID); } /* define instance of prte_class_t */ PMIX_CLASS_INSTANCE(prte_namelist_t, /* type name */ pmix_list_item_t, /* parent "class" name */ prte_namelist_construct, /* constructor */ NULL); /* destructor */ static bool fns_init = false; static pmix_tsd_key_t print_args_tsd_key; char *prte_print_args_null = "NULL"; typedef struct { char *buffers[PRTE_PRINT_NAME_ARG_NUM_BUFS]; int cntr; } prte_print_args_buffers_t; static void buffer_cleanup(void *value) { int i; prte_print_args_buffers_t *ptr; if (NULL != value) { ptr = (prte_print_args_buffers_t *) value; for (i = 0; i < PRTE_PRINT_NAME_ARG_NUM_BUFS; i++) { free(ptr->buffers[i]); } free(ptr); } } static prte_print_args_buffers_t *get_print_name_buffer(void) { prte_print_args_buffers_t *ptr; int ret, i; if (!fns_init) { /* setup the print_args function */ if (PRTE_SUCCESS != (ret = pmix_tsd_key_create(&print_args_tsd_key, buffer_cleanup))) { PRTE_ERROR_LOG(ret); return NULL; } fns_init = true; } ret = pmix_tsd_getspecific(print_args_tsd_key, (void **) &ptr); if (PRTE_SUCCESS != ret) return NULL; if (NULL == ptr) { ptr = (prte_print_args_buffers_t *) malloc(sizeof(prte_print_args_buffers_t)); for (i = 0; i < PRTE_PRINT_NAME_ARG_NUM_BUFS; i++) { ptr->buffers[i] = (char *) malloc((PRTE_PRINT_NAME_ARGS_MAX_SIZE + 1) * sizeof(char)); } ptr->cntr = 0; ret = pmix_tsd_setspecific(print_args_tsd_key, (void *) ptr); } return (prte_print_args_buffers_t *) ptr; } char *prte_util_print_name_args(const pmix_proc_t *name) { prte_print_args_buffers_t *ptr; char *job, *vpid; /* protect against NULL names */ if (NULL == name) { /* get the next buffer */ ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "[NO-NAME]"); ptr->cntr++; return ptr->buffers[ptr->cntr - 1]; } /* get the jobid, vpid strings first - this will protect us from * stepping on each other's buffer. This also guarantees * that the print_args function has been initialized, so * we don't need to duplicate that here */ job = prte_util_print_jobids(name->nspace); vpid = prte_util_print_vpids(name->rank); /* get the next buffer */ ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "[%s,%s]", job, vpid); ptr->cntr++; return ptr->buffers[ptr->cntr - 1]; } char *prte_util_print_jobids(const pmix_nspace_t job) { prte_print_args_buffers_t *ptr; ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } if (0 == strlen(job)) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "[INVALID]"); } else { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", job); } ptr->cntr++; return ptr->buffers[ptr->cntr - 1]; } char *prte_util_print_job_family(const pmix_nspace_t job) { prte_print_args_buffers_t *ptr; char *cptr; ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } /* see if the job is invalid */ if (PMIX_NSPACE_INVALID(job)) { snprintf(ptr->buffers[ptr->cntr++], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "[INVALID]"); } else { /* find the '@' sign delimiting the job family */ cptr = strrchr(job, '@'); if (NULL == cptr) { /* this isn't a PRRTE job */ snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", job); } else { *cptr = '\0'; snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", job); *cptr = '@'; } ptr->cntr++; } return ptr->buffers[ptr->cntr - 1]; } char *prte_util_print_local_jobid(const pmix_nspace_t job) { prte_print_args_buffers_t *ptr; char *cptr; ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } /* see if the job is invalid */ if (PMIX_NSPACE_INVALID(job)) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "[INVALID]"); ptr->cntr++; } else { /* find the '@' sign delimiting the job family */ cptr = strrchr(job, '@'); if (NULL == cptr) { /* this isn't a PRRTE job */ snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", job); } else { ++cptr; snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", cptr); } ptr->cntr++; } return ptr->buffers[ptr->cntr - 1]; } char *prte_util_print_vpids(const pmix_rank_t vpid) { prte_print_args_buffers_t *ptr; ptr = get_print_name_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_print_args_null; } /* cycle around the ring */ if (PRTE_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } if (PMIX_RANK_INVALID == vpid) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "INVALID"); } else if (PMIX_RANK_WILDCARD == vpid) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "WILDCARD"); } else if (PMIX_RANK_LOCAL_NODE == vpid) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "LOCALNODE"); } else if (PMIX_RANK_LOCAL_PEERS == vpid) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "LOCALPEERS"); } else if (PMIX_RANK_UNDEF == vpid) { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%s", "UNDEFINED"); } else { snprintf(ptr->buffers[ptr->cntr], PRTE_PRINT_NAME_ARGS_MAX_SIZE, "%u", vpid); } ptr->cntr++; return ptr->buffers[ptr->cntr - 1]; } /*** STRING FUNCTIONS ***/ int prte_util_convert_vpid_to_string(char **vpid_string, const pmix_rank_t vpid) { /* check for wildcard value - handle appropriately */ if (PMIX_RANK_WILDCARD == vpid) { *vpid_string = strdup("WILDCARD"); } else if (PMIX_RANK_INVALID == vpid) { *vpid_string = strdup("INVALID"); } else if (PMIX_RANK_LOCAL_NODE == vpid) { *vpid_string = strdup("LOCALNODE"); } else if (PMIX_RANK_LOCAL_PEERS == vpid) { *vpid_string = strdup("LOCALPEERS"); } else if (PMIX_RANK_UNDEF == vpid) { *vpid_string = strdup("UNDEFINED"); } else { if (0 > pmix_asprintf(vpid_string, "%u", vpid)) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } } return PRTE_SUCCESS; } int prte_util_convert_string_to_process_name(pmix_proc_t *name, const char *name_string) { char *p; /* check for NULL string - error */ if (NULL == name_string) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } p = strrchr(name_string, '.'); /** get last field -> vpid */ /* check for error */ if (NULL == p) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } *p = '\0'; PMIX_LOAD_NSPACE(name->nspace, name_string); *p = '.'; ++p; name->rank = strtoul(p, NULL, 10); return PRTE_SUCCESS; } int prte_util_convert_process_name_to_string(char **name_string, const pmix_proc_t *name) { char *job, *rank; if (NULL == name) { /* got an error */ PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } job = prte_util_print_jobids(name->nspace); rank = prte_util_print_vpids(name->rank); pmix_asprintf(name_string, "%s.%s", job, rank); return PRTE_SUCCESS; } /**** COMPARE NAME FIELDS ****/ int prte_util_compare_name_fields(prte_ns_cmp_bitmask_t fields, const pmix_proc_t *name1, const pmix_proc_t *name2) { /* handle the NULL pointer case */ if (NULL == name1 && NULL == name2) { return PRTE_EQUAL; } else if (NULL == name1) { return PRTE_VALUE2_GREATER; } else if (NULL == name2) { return PRTE_VALUE1_GREATER; } /* in this comparison function, we check for exact equalities. * In the case of wildcards, we check to ensure that the fields * actually match those values - thus, a "wildcard" in this * function does not actually stand for a wildcard value, but * rather a specific value - UNLESS the CMP_WILD bitmask value * is set */ /* check job id */ if (PRTE_NS_CMP_JOBID & fields) { if (PRTE_NS_CMP_WILD & fields && (0 == strlen(name1->nspace) || 0 == strlen(name2->nspace))) { goto check_vpid; } if (strlen(name1->nspace) < strlen(name2->nspace)) { return PRTE_VALUE2_GREATER; } else if (strlen(name1->nspace) > strlen(name2->nspace)) { return PRTE_VALUE1_GREATER; } } /* get here if jobid's are equal, or not being checked * now check vpid */ check_vpid: if (PRTE_NS_CMP_VPID & fields) { if (PRTE_NS_CMP_WILD & fields && (PMIX_RANK_WILDCARD == name1->rank || PMIX_RANK_WILDCARD == name2->rank)) { return PRTE_EQUAL; } if (name1->rank < name2->rank) { return PRTE_VALUE2_GREATER; } else if (name1->rank > name2->rank) { return PRTE_VALUE1_GREATER; } } /* only way to get here is if all fields are being checked and are equal, * or jobid not checked, but vpid equal, * only vpid being checked, and equal * return that fact */ return PRTE_EQUAL; } char *prte_pretty_print_timing(int64_t secs, int64_t usecs) { unsigned long minutes, seconds; float fsecs; char *timestring; seconds = secs + (usecs / 1000000l); minutes = seconds / 60l; seconds = seconds % 60l; if (0 == minutes && 0 == seconds) { fsecs = ((float) (secs) *1000000.0 + (float) usecs) / 1000.0; pmix_asprintf(×tring, "%8.2f millisecs", fsecs); } else { pmix_asprintf(×tring, "%3lu:%02lu min:sec", minutes, seconds); } return timestring; } char *prte_util_make_version_string(const char *scope, int major, int minor, int release, const char *greek, const char *repo) { char *str = NULL, *tmp; char temp[BUFSIZ]; temp[BUFSIZ - 1] = '\0'; if (0 == strcmp(scope, "full") || 0 == strcmp(scope, "all")) { snprintf(temp, BUFSIZ - 1, "%d.%d", major, minor); str = strdup(temp); if (release >= 0) { snprintf(temp, BUFSIZ - 1, ".%d", release); pmix_asprintf(&tmp, "%s%s", str, temp); free(str); str = tmp; } if (NULL != greek) { pmix_asprintf(&tmp, "%s%s", str, greek); free(str); str = tmp; } if (NULL != repo) { pmix_asprintf(&tmp, "%s%s", str, repo); free(str); str = tmp; } } else if (0 == strcmp(scope, "major")) { snprintf(temp, BUFSIZ - 1, "%d", major); } else if (0 == strcmp(scope, "minor")) { snprintf(temp, BUFSIZ - 1, "%d", minor); } else if (0 == strcmp(scope, "release")) { snprintf(temp, BUFSIZ - 1, "%d", release); } else if (0 == strcmp(scope, "greek")) { str = strdup(greek); } else if (0 == strcmp(scope, "repo")) { str = strdup(repo); } if (NULL == str) { str = strdup(temp); } return str; } prrte-4.1.1~rc2/src/runtime/0000775000175000017500000000000015203542132016111 5ustar alastairalastairprrte-4.1.1~rc2/src/runtime/data_type_support/0000775000175000017500000000000015203542132021657 5ustar alastairalastairprrte-4.1.1~rc2/src/runtime/data_type_support/prte_dt_copy_fns.c0000664000175000017500000001056615203542132025374 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #include #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" /** * JOB */ int prte_job_copy(prte_job_t **dest, prte_job_t *src) { (*dest) = src; PMIX_RETAIN(src); return PRTE_SUCCESS; } /** * NODE */ int prte_node_copy(prte_node_t **dest, prte_node_t *src) { prte_node_t *node; node = PMIX_NEW(prte_node_t); node->name = strdup(src->name); node->state = src->state; node->slots = src->slots; node->slots_inuse = src->slots_inuse; node->slots_max = src->slots_max; node->topology = src->topology; node->flags = src->flags; (*dest) = node; return PRTE_SUCCESS; } /** * PROC */ int prte_proc_copy(prte_proc_t **dest, prte_proc_t *src) { (*dest) = src; PMIX_RETAIN(src); return PRTE_SUCCESS; } /* * APP CONTEXT */ int prte_app_copy(prte_app_context_t **dest, prte_app_context_t *src) { prte_value_t *kv, *kvnew; pmix_status_t rc; /* create the new object */ *dest = PMIX_NEW(prte_app_context_t); if (NULL == *dest) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* copy data into it */ (*dest)->idx = src->idx; if (NULL != src->app) { (*dest)->app = strdup(src->app); } (*dest)->num_procs = src->num_procs; (*dest)->argv = PMIx_Argv_copy(src->argv); (*dest)->env = PMIx_Argv_copy(src->env); if (NULL != src->cwd) { (*dest)->cwd = strdup(src->cwd); } PMIX_LIST_FOREACH(kv, &src->attributes, prte_value_t) { kvnew = PMIX_NEW(prte_value_t); PMIX_VALUE_XFER_DIRECT(rc, &kvnew->value, &kv->value); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(kvnew); return prte_pmix_convert_status(rc); } pmix_list_append(&(*dest)->attributes, &kvnew->super); } return PRTE_SUCCESS; } /* * JOB_MAP */ int prte_map_copy(struct prte_job_map_t **d, struct prte_job_map_t *s) { int32_t i; prte_job_map_t **dest = (prte_job_map_t **) d; prte_job_map_t *src = (prte_job_map_t *) s; if (NULL == src) { *dest = NULL; return PRTE_SUCCESS; } /* create the new object */ *dest = PMIX_NEW(prte_job_map_t); if (NULL == *dest) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* copy data into it */ (*dest)->mapping = src->mapping; (*dest)->ranking = src->ranking; (*dest)->binding = src->binding; (*dest)->num_new_daemons = src->num_new_daemons; (*dest)->daemon_vpid_start = src->daemon_vpid_start; (*dest)->num_nodes = src->num_nodes; /* copy the pointer array - have to do this manually * as no dss.copy function is setup for that object */ (*dest)->nodes->lowest_free = src->nodes->lowest_free; (*dest)->nodes->number_free = src->nodes->number_free; (*dest)->nodes->size = src->nodes->size; (*dest)->nodes->max_size = src->nodes->max_size; (*dest)->nodes->block_size = src->nodes->block_size; for (i = 0; i < src->nodes->size; i++) { (*dest)->nodes->addr[i] = src->nodes->addr[i]; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/data_type_support/prte_dt_print_fns.c0000664000175000017500000005737615203542132025570 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "types.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/runtime/prte_globals.h" #include "src/util/error_strings.h" #include "src/util/name_fns.h" /* This function is a modified version of the one found in src/mca/ras/base/ras_base_allocate.c*/ static void display_cpus(prte_topology_t *t, prte_job_t *jdata, char *node, char**output) { char tmp[2048]; unsigned pkg, npkgs; bool bits_as_cores = false, use_hwthread_cpus = prte_hwloc_default_use_hwthread_cpus; unsigned npus, ncores; hwloc_obj_t obj; hwloc_cpuset_t avail = NULL; hwloc_cpuset_t allowed; hwloc_cpuset_t coreset = NULL; PRTE_HIDE_UNUSED_PARAMS(node); char *tmp1, *tmp2; npus = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_PU); ncores = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_CORE); if (npus == ncores && !use_hwthread_cpus) { /* the bits in this bitmap represent cores */ bits_as_cores = true; } use_hwthread_cpus = prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL); if (!use_hwthread_cpus && !bits_as_cores) { coreset = hwloc_bitmap_alloc(); } avail = hwloc_bitmap_alloc(); pmix_asprintf(&tmp1, " \n"); npkgs = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_PACKAGE); allowed = (hwloc_cpuset_t)hwloc_topology_get_allowed_cpuset(t->topo); for (pkg = 0; pkg < npkgs; pkg++) { obj = prte_hwloc_base_get_obj_by_type(t->topo, HWLOC_OBJ_PACKAGE, pkg); hwloc_bitmap_and(avail, obj->cpuset, allowed); if (hwloc_bitmap_iszero(avail)) { pmix_asprintf(&tmp2, "%s \n", tmp1, pkg, "NONE"); continue; } if (bits_as_cores) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); pmix_asprintf(&tmp2, "%s \n", tmp1, pkg, tmp); } else if (use_hwthread_cpus) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); pmix_asprintf(&tmp2, "%s \n", tmp1, pkg, tmp); } else { prte_hwloc_build_map(t->topo, avail, use_hwthread_cpus | bits_as_cores, coreset); /* now print out the string */ hwloc_bitmap_list_snprintf(tmp, 2048, coreset); pmix_asprintf(&tmp2, "%s \n", tmp1, pkg, tmp); } free(tmp1); tmp1 = tmp2; tmp2 = NULL; } hwloc_bitmap_free(avail); if (NULL != coreset) { hwloc_bitmap_free(coreset); } pmix_asprintf(output, "%s \n", tmp1); free(tmp1); return; } /* * JOB */ void prte_job_print(char **output, prte_job_t *src) { char *tmp, *tmp2, *tmp3; int32_t i; prte_app_context_t *app; prte_proc_t *proc; /* set default result */ *output = NULL; tmp2 = PMIx_Argv_join(src->personality, ','); pmix_asprintf(&tmp, "\nData for job: %s\tPersonality: %s\tRecovery: %s\n\tNum apps: %ld\tStdin " "target: %s\tState: %s\tAbort: %s", PRTE_JOBID_PRINT(src->nspace), tmp2, (prte_get_attribute(&src->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL)) ? "ENABLED" : "DISABLED", (long) src->num_apps, PRTE_VPID_PRINT(src->stdin_target), prte_job_state_to_str(src->state), (PRTE_FLAG_TEST(src, PRTE_JOB_FLAG_ABORTED)) ? "True" : "False"); free(tmp2); for (i = 0; i < src->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(src->apps, i))) { continue; } prte_app_print(&tmp2, src, app); pmix_asprintf(&tmp3, "%s\n%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } if (NULL != src->map) { prte_map_print(&tmp2, src); pmix_asprintf(&tmp3, "%s%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } else { pmix_asprintf(&tmp2, "%s\nNo Map", tmp); free(tmp); tmp = tmp2; } pmix_asprintf(&tmp2, "%s\nNum procs: %ld\tOffset: %ld", tmp, (long) src->num_procs, (long) src->offset); free(tmp); tmp = tmp2; for (i = 0; i < src->procs->size; i++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(src->procs, i))) { continue; } prte_proc_print(&tmp2, src, proc); pmix_asprintf(&tmp3, "%s%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } pmix_asprintf(&tmp2, "%s\n\tNum launched: %ld\tNum reported: %ld\tNum terminated: %ld", tmp, (long) src->num_launched, (long) src->num_reported, (long) src->num_terminated); free(tmp); tmp = tmp2; /* set the return */ *output = tmp; return; } /* * NODE */ void prte_node_print(char **output, prte_job_t *jdata, prte_node_t *src) { char *tmp, *tmp1, *tmp2, *tmp3; int32_t i,j; prte_proc_t *proc; prte_topology_t *t; /* set default result */ *output = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT, NULL, PMIX_BOOL)) { pmix_asprintf(&tmp, " \n", (NULL == src->name) ? "UNKNOWN" : src->name, (int) src->slots, (int) src->slots_max); tmp2 = (char*)malloc(1); tmp2[0] = '\0'; for (j=0; j < prte_node_topologies->size; j++) { t = (prte_topology_t*)pmix_pointer_array_get_item(prte_node_topologies, j); if (NULL != t) { display_cpus(t, jdata, "N/A", &tmp1); pmix_asprintf(&tmp3, "%s%s",tmp2, tmp1); free(tmp1); tmp1 = NULL; free(tmp2); tmp2 = NULL; tmp2 = tmp3; } } pmix_asprintf(&tmp3, "%s%s", tmp,tmp2); free(tmp2); tmp2 = NULL; free(tmp1); tmp1 = NULL; free(tmp); tmp = tmp3; /* loop through procs and print their rank */ for (j = 0; j < src->procs->size; j++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(src->procs, j))) { continue; } if (!PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) { continue; } prte_proc_print(&tmp2, jdata, proc); pmix_asprintf(&tmp3, "%s%s", tmp, tmp2); free(tmp2); tmp2 = NULL; free(tmp); tmp = tmp3; } pmix_asprintf(&tmp3, "%s \n", tmp); free(tmp); *output = tmp3; return; } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL)) { /* just provide a simple output for users */ pmix_asprintf(&tmp, "\nData for node: %s\tNum slots: %ld\tMax slots: %ld\tNum procs: %ld", (NULL == src->name) ? "UNKNOWN" : src->name, (long) src->slots, (long) src->slots_max, (long) src->num_procs); if (0 == src->num_procs) { *output = tmp; return; } goto PRINT_PROCS; } tmp3 = prte_ras_base_flag_string(src); pmix_asprintf(&tmp, "\nData for node: %s\tState: %0x\t%s", (NULL == src->name) ? "UNKNOWN" : src->name, src->state, tmp3); free(tmp3); tmp3 = NULL; /* does this node have any aliases? */ if (NULL != src->aliases) { for (i = 0; NULL != src->aliases[i]; i++) { pmix_asprintf(&tmp2, "%s\n resolved from %s", tmp, src->aliases[i]); free(tmp); tmp = tmp2; } } pmix_asprintf(&tmp2, "%s\n Daemon: %s\tDaemon launched: %s", tmp, (NULL == src->daemon) ? "Not defined" : PRTE_NAME_PRINT(&(src->daemon->name)), PRTE_FLAG_TEST(src, PRTE_NODE_FLAG_DAEMON_LAUNCHED) ? "True" : "False"); free(tmp); tmp = tmp2; pmix_asprintf(&tmp2, "%s\n Num slots: %ld\tSlots in use: %ld\tOversubscribed: %s", tmp, (long) src->slots, (long) src->slots_inuse, PRTE_FLAG_TEST(src, PRTE_NODE_FLAG_OVERSUBSCRIBED) ? "TRUE" : "FALSE"); free(tmp); tmp = tmp2; pmix_asprintf(&tmp2, "%s\n Num slots allocated: %ld\tMax slots: %ld\tNum procs: %ld", tmp, (long) src->slots, (long) src->slots_max, (long) src->num_procs); free(tmp); tmp = tmp2; tmp3 = NULL; if (prte_get_attribute(&src->attributes, PRTE_NODE_USERNAME, (void **) &tmp3, PMIX_STRING) && NULL != tmp3) { pmix_asprintf(&tmp2, "%s\n Username on node: %s", tmp, tmp3); free(tmp3); free(tmp); tmp = tmp2; } PRINT_PROCS: /* we want to print these procs in their job-rank'd order, but they * will be in the node array based on the order in which they were * mapped - which doesn't match job-rank'd order in many cases */ for (i = 0; i < jdata->procs->size; i++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, i))) { continue; } if (proc->node != src) { continue; } prte_proc_print(&tmp2, jdata, proc); pmix_asprintf(&tmp3, "%s%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } /* set the return */ *output = tmp; return; } /* * PROC */ void prte_proc_print(char **output, prte_job_t *jdata, prte_proc_t *src) { char *tmp, *tmp3, *tmp4, *pfx2 = " "; char *tmp2; hwloc_cpuset_t mycpus; char *str; bool use_hwthread_cpus; int pkgnum; int npus; char *cores = NULL; char xmlsp = ' '; bool physical; /* set default result */ *output = NULL; /* check for type of cpu being used */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { use_hwthread_cpus = true; } else { use_hwthread_cpus = false; } physical = prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_PHYSICAL_CPUS, NULL, PMIX_BOOL); if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT, NULL, PMIX_BOOL)) { if (NULL != src->cpuset && NULL != src->node->topology && NULL != src->node->topology->topo) { mycpus = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(mycpus, src->cpuset); npus = prte_hwloc_base_get_nbobjs_by_type(src->node->topology->topo, HWLOC_OBJ_PU); /* assuming each "core" xml element will take 20 characters. There could be at most npus such elements */ int sz = sizeof(char) * npus * 20; cores = (char*)malloc(sz); if (NULL == cores) { pmix_asprintf(&tmp, "\n%*c\n", 8, xmlsp); *output = tmp; return; } prte_hwloc_get_binding_info(mycpus, use_hwthread_cpus, src->node->topology->topo, &pkgnum, cores, sz); hwloc_bitmap_free(mycpus); pmix_asprintf(&tmp, "\n%*c\n%*c\n" "%*c\n%s\n%*c\n%*c\n%*c\n", 8, xmlsp, PRTE_VPID_PRINT(src->name.rank), (long) src->app_idx, 12, xmlsp, 16, xmlsp, pkgnum, cores, 16, xmlsp, 12, xmlsp, 8, xmlsp); free (cores); } else { pmix_asprintf(&tmp, "\n%*c\n%*c\n%*c\n", 8, xmlsp, PRTE_VPID_PRINT(src->name.rank), 12, xmlsp, 8, xmlsp); } /* set the return */ *output = tmp; return; } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL)) { if (NULL != src->cpuset && NULL != src->node->topology && NULL != src->node->topology->topo) { mycpus = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(mycpus, src->cpuset); str = prte_hwloc_base_cset2str(mycpus, use_hwthread_cpus, physical, src->node->topology->topo); if (NULL == str) { str = strdup("UNBOUND"); } hwloc_bitmap_free(mycpus); pmix_asprintf(&tmp, "\n%sProcess jobid: %s App: %ld Process rank: %s Bound: %s", pfx2, PRTE_JOBID_PRINT(src->name.nspace), (long) src->app_idx, PRTE_VPID_PRINT(src->name.rank), str); free(str); } else { /* just print a very simple output for users */ pmix_asprintf(&tmp, "\n%sProcess jobid: %s App: %ld Process rank: %s Bound: N/A", pfx2, PRTE_JOBID_PRINT(src->name.nspace), (long) src->app_idx, PRTE_VPID_PRINT(src->name.rank)); } /* set the return */ *output = tmp; return; } pmix_asprintf(&tmp, "\n%sData for proc: %s", pfx2, PRTE_NAME_PRINT(&src->name)); pmix_asprintf(&tmp3, "%s\n%s Pid: %ld\tLocal rank: %lu\tNode rank: %lu\tApp rank: %d", tmp, pfx2, (long) src->pid, (unsigned long) src->local_rank, (unsigned long) src->node_rank, src->app_rank); free(tmp); tmp = tmp3; if (NULL != src->cpuset) { mycpus = hwloc_bitmap_alloc(); hwloc_bitmap_list_sscanf(mycpus, src->cpuset); tmp2 = prte_hwloc_base_cset2str(mycpus, use_hwthread_cpus, physical, src->node->topology->topo); hwloc_bitmap_free(mycpus); } else { tmp2 = strdup("UNBOUND"); } pmix_asprintf(&tmp4, "%s\n%s State: %s\tApp_context: %ld\n%s\tBinding: %s", tmp, pfx2, prte_proc_state_to_str(src->state), (long) src->app_idx, pfx2, tmp2); free(tmp); free(tmp2); /* set the return */ *output = tmp4; return; } /* * APP CONTEXT */ void prte_app_print(char **output, prte_job_t *jdata, prte_app_context_t *src) { char *tmp, *tmp2, *tmp3; int i, count; PRTE_HIDE_UNUSED_PARAMS(jdata); /* set default result */ *output = NULL; pmix_asprintf(&tmp, "\nData for app_context: index %lu\tapp: %s\n\tNum procs: %lu\tFirstRank: %s", (unsigned long) src->idx, (NULL == src->app) ? "NULL" : src->app, (unsigned long) src->num_procs, PRTE_VPID_PRINT(src->first_rank)); count = PMIx_Argv_count(src->argv); for (i = 0; i < count; i++) { pmix_asprintf(&tmp2, "%s\n\tArgv[%d]: %s", tmp, i, src->argv[i]); free(tmp); tmp = tmp2; } count = PMIx_Argv_count(src->env); for (i = 0; i < count; i++) { pmix_asprintf(&tmp2, "%s\n\tEnv[%lu]: %s", tmp, (unsigned long) i, src->env[i]); free(tmp); tmp = tmp2; } tmp3 = NULL; for (i=0; NULL != src->env && NULL != src->env[i]; i++) { if (0 == strncmp(src->env[i], "PMIX_PREFIX", strlen("PMIX_PREFIX"))) { tmp3 = src->env[i]; tmp3 += strlen("PMIX_PREFIX="); } } pmix_asprintf(&tmp2, "%s\n\tWorking dir: %s\n\tPMIxPrefix: %s\n\tUsed on node: %s", tmp, (NULL == src->cwd) ? "NULL" : src->cwd, (NULL == tmp3) ? "NULL" : tmp3, PRTE_FLAG_TEST(src, PRTE_APP_FLAG_USED_ON_NODE) ? "TRUE" : "FALSE"); free(tmp); tmp = tmp2; /* set the return */ *output = tmp; return; } /* * JOB_MAP */ void prte_map_print(char **output, prte_job_t *jdata) { char *tmp = NULL, *tmp2 = NULL, *tmp3 = NULL, *tmp4 = NULL; char *tmp_node = NULL; int32_t i; prte_node_t *node; prte_job_map_t *src = jdata->map; uint16_t u16, *u16ptr = &u16; char *ppr, *cpus_per_rank, *cpu_type, *cpuset = NULL; /* set default result */ *output = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT, NULL, PMIX_BOOL)) { /* creating the output in an XML format */ pmix_asprintf(&tmp4, "\n\n"); tmp = (char*)malloc(1); tmp[0] = '\0'; /* loop through nodes */ for (i = 0; i < src->nodes->size; i++) { if (NULL == (node = (prte_node_t*)pmix_pointer_array_get_item(src->nodes, i))) { continue; } prte_node_print(&tmp_node, jdata, node); pmix_asprintf(&tmp3, "%s%s",tmp,tmp_node); free(tmp_node); tmp_node = NULL; free(tmp); tmp = tmp3; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { pmix_asprintf(&tmp2, "%s\n", tmp); free(tmp); tmp = tmp2; } /* end of the xml "map" tag */ pmix_asprintf(&tmp2, "%s%s\n", tmp4,tmp); *output = tmp2; free(tmp); free(tmp4); return; } ppr = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PPR, (void **) &ppr, PMIX_STRING) || NULL == ppr) { ppr = strdup("N/A"); } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, (void **) &u16ptr, PMIX_UINT16)) { pmix_asprintf(&cpus_per_rank, "%d", (int) u16); } else { cpus_per_rank = strdup("N/A"); } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { cpu_type = "HWT"; } else { cpu_type = "CORE"; } cpuset = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_CPUSET, (void **) &cpuset, PMIX_STRING)) { if (NULL == prte_hwloc_default_cpu_list) { cpuset = strdup("N/A"); } else { cpuset = strdup(prte_hwloc_default_cpu_list); } } else if (NULL == cpuset) { cpuset = strdup("N/A"); } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL)) { pmix_asprintf( &tmp, "\n================================= JOB MAP =================================\n" "Data for JOB %s offset %s Total slots allocated %lu\n" "Mapper requested: %s Last mapper: %s Mapping policy: %s Ranking policy: %s\n" "Binding policy: %s Cpu set: %s PPR: %s Cpus-per-rank: %s Cpu Type: %s", PRTE_JOBID_PRINT(jdata->nspace), PRTE_VPID_PRINT(jdata->offset), (long unsigned) jdata->total_slots_alloc, (NULL == src->req_mapper) ? "NULL" : src->req_mapper, (NULL == src->last_mapper) ? "NULL" : src->last_mapper, prte_rmaps_base_print_mapping(src->mapping), prte_rmaps_base_print_ranking(src->ranking), prte_hwloc_base_print_binding(src->binding), cpuset, ppr, cpus_per_rank, cpu_type); if (PMIX_RANK_INVALID == src->daemon_vpid_start) { pmix_asprintf( &tmp2, "%s\nNum new daemons: %ld\tNew daemon starting vpid INVALID\nNum nodes: %ld", tmp, (long) src->num_new_daemons, (long) src->num_nodes); } else { pmix_asprintf(&tmp2, "%s\nNum new daemons: %ld\tNew daemon starting vpid %ld\nNum nodes: %ld", tmp, (long) src->num_new_daemons, (long) src->daemon_vpid_start, (long) src->num_nodes); } free(tmp); tmp = tmp2; } else { /* this is being printed for a user, so let's make it easier to see */ pmix_asprintf(&tmp, "\n======================== JOB MAP ========================\n" "Data for JOB %s offset %s Total slots allocated %lu\n" " Mapping policy: %s Ranking policy: %s Binding policy: %s\n" " Cpu set: %s PPR: %s Cpus-per-rank: %s Cpu Type: %s\n", PRTE_JOBID_PRINT(jdata->nspace), PRTE_VPID_PRINT(jdata->offset), (long unsigned) jdata->total_slots_alloc, prte_rmaps_base_print_mapping(src->mapping), prte_rmaps_base_print_ranking(src->ranking), prte_hwloc_base_print_binding(src->binding), cpuset, ppr, cpus_per_rank, cpu_type); } free(ppr); free(cpus_per_rank); free(cpuset); for (i = 0; i < src->nodes->size; i++) { if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(src->nodes, i))) { continue; } prte_node_print(&tmp2, jdata, node); pmix_asprintf(&tmp3, "%s\n%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } /* put some warning out for the donotlaunch case */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { pmix_asprintf(&tmp2, "%s\n\nWarning: This map has been generated with the DONOTLAUNCH option;\n" "\tThe compute node architecture has not been probed, and the displayed\n" "\tmap reflects the HEADNODE ARCHITECTURE. On systems with a different\n" "\tarchitecture between headnode and compute nodes, the map can be\n" "\tdisplayed using `prte --display map /bin/true`, which will launch\n" "\tenough of the DVM to probe the compute node architecture.", tmp); free(tmp); tmp = tmp2; } /* let's make it easier to see */ pmix_asprintf(&tmp2, "%s\n\n=============================================================\n", tmp); free(tmp); tmp = tmp2; /* set the return */ *output = tmp; return; } prrte-4.1.1~rc2/src/runtime/data_type_support/prte_dt_packing_fns.c0000664000175000017500000004525015203542132026034 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "types.h" #include #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/runtime/prte_globals.h" /* * JOB * NOTE: We do not pack all of the job object's fields as many of them have no * value in sending them to another location. The only purpose in packing and * sending a job object is to communicate the data required to dynamically * spawn another job - so we only pack that limited set of required data */ int prte_job_pack(pmix_data_buffer_t *bkt, prte_job_t *job) { pmix_status_t rc; int32_t j, count, bookmark; prte_app_context_t *app; prte_proc_t *proc; prte_attribute_t *kv; pmix_list_t *cache; prte_info_item_t *val; /* pack the nspace */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the flags */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->flags, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the attributes that need to be sent */ count = 0; PMIX_LIST_FOREACH(kv, &job->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { ++count; } } rc = PMIx_Data_pack(NULL, bkt, (void *) &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } PMIX_LIST_FOREACH(kv, &job->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->key, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->data, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } /* check for job info attribute */ cache = NULL; if (prte_get_attribute(&job->attributes, PRTE_JOB_INFO_CACHE, (void **) &cache, PMIX_POINTER) && NULL != cache) { /* we need to pack these as well, but they are composed * of prte_info_item_t's on a list. So first pack the number * of list elements */ count = pmix_list_get_size(cache); rc = PMIx_Data_pack(NULL, bkt, (void *) &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* now pack each element on the list */ PMIX_LIST_FOREACH(val, cache, prte_info_item_t) { rc = PMIx_Data_pack(NULL, bkt, (void *) &val->info, 1, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } else { /* pack a zero to indicate no job info is being passed */ count = 0; rc = PMIx_Data_pack(NULL, bkt, (void *) &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } /* pack the personality */ count = PMIx_Argv_count(job->personality); rc = PMIx_Data_pack(NULL, bkt, (void *) &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } for (j = 0; j < count; j++) { rc = PMIx_Data_pack(NULL, bkt, (void *) &job->personality[j], 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } /* pack the number of apps */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->num_apps, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* if there are apps, pack the app_contexts */ if (0 < job->num_apps) { for (j = 0; j < job->apps->size; j++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(job->apps, j))) { continue; } rc = prte_app_pack(bkt, app); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } /* pack the number of procs and offset */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->num_procs, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, (void *) &job->offset, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < job->num_procs) { for (j = 0; j < job->procs->size; j++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(job->procs, j))) { continue; } rc = prte_proc_pack(bkt, proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } /* pack the stdin target */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->stdin_target, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the total slots allocated to the job */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->total_slots_alloc, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* if the map is NULL, then we cannot pack it as there is * nothing to pack. However, we have to flag whether or not * the map is included so the unpacking routine can know * what to do */ if (NULL == job->map) { /* pack a zero value */ j = 0; } else { /* pack a one to indicate a map is there */ j = 1; } rc = PMIx_Data_pack(NULL, bkt, (void *) &j, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the map - this will only pack the fields that control * HOW a job is to be mapped. We do -not- pack the mapped procs * or nodes as this info does not need to be transmitted */ if (NULL != job->map) { rc = prte_map_pack(bkt, job->map); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } /* pack the bookmark */ if (NULL == job->bookmark) { bookmark = -1; } else { bookmark = job->bookmark->index; } rc = PMIx_Data_pack(NULL, bkt, (void *) &bookmark, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the job state */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->state, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the launcher ID */ rc = PMIx_Data_pack(NULL, bkt, (void *) &job->launcher, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } return PRTE_SUCCESS; } int prte_node_pack(pmix_data_buffer_t *bkt, prte_node_t *node) { int rc; int32_t count; uint8_t flag; prte_attribute_t *kv; /* do not pack the index - it is meaningless on the other end */ /* pack the node name */ rc = PMIx_Data_pack(NULL, bkt, (void *) &node->name, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* do not pack the daemon name or launch id */ /* pack the number of procs on the node */ rc = PMIx_Data_pack(NULL, bkt, (void *) &node->num_procs, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* do not pack the procs */ /* pack whether we are oversubscribed or not */ flag = PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_OVERSUBSCRIBED); rc = PMIx_Data_pack(NULL, bkt, (void *) &flag, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the state */ rc = PMIx_Data_pack(NULL, bkt, (void *) &node->state, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack any shared attributes */ count = 0; PMIX_LIST_FOREACH(kv, &node->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { ++count; } } rc = PMIx_Data_pack(NULL, bkt, (void *) &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < count) { PMIX_LIST_FOREACH(kv, &node->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->key, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->data, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } } return PRTE_SUCCESS; } /* * PROC */ int prte_proc_pack(pmix_data_buffer_t *bkt, prte_proc_t *proc) { pmix_status_t rc; int32_t count; prte_attribute_t *kv; /* pack the name */ rc = PMIx_Data_pack(NULL, bkt, &proc->name, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the daemon/node it is on */ rc = PMIx_Data_pack(NULL, bkt, &proc->parent, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the local rank */ rc = PMIx_Data_pack(NULL, bkt, &proc->local_rank, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the node rank */ rc = PMIx_Data_pack(NULL, bkt, &proc->node_rank, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the state */ rc = PMIx_Data_pack(NULL, bkt, &proc->state, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the app context index */ rc = PMIx_Data_pack(NULL, bkt, &proc->app_idx, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the app rank */ rc = PMIx_Data_pack(NULL, bkt, &proc->app_rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the cpuset */ rc = PMIx_Data_pack(NULL, bkt, (void *) &proc->cpuset, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the attributes that will go */ count = 0; PMIX_LIST_FOREACH(kv, &proc->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { ++count; } } rc = PMIx_Data_pack(NULL, bkt, &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < count) { PMIX_LIST_FOREACH(kv, &proc->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->key, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->data, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } } return PRTE_SUCCESS; } /* * APP CONTEXT */ int prte_app_pack(pmix_data_buffer_t *bkt, prte_app_context_t *app) { pmix_status_t rc; int32_t count, j; prte_attribute_t *kv; /* pack the application index (for multiapp jobs) */ rc = PMIx_Data_pack(NULL, bkt, &app->idx, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the application name */ rc = PMIx_Data_pack(NULL, bkt, &app->app, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the number of processes */ rc = PMIx_Data_pack(NULL, bkt, &app->num_procs, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the first rank for this app */ rc = PMIx_Data_pack(NULL, bkt, &app->first_rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the number of entries in the argv array */ count = PMIx_Argv_count(app->argv); rc = PMIx_Data_pack(NULL, bkt, &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* if there are entries, pack the argv entries */ for (j = 0; j < count; j++) { rc = PMIx_Data_pack(NULL, bkt, (void *) &app->argv[j], 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } /* pack the number of entries in the enviro array */ count = PMIx_Argv_count(app->env); rc = PMIx_Data_pack(NULL, bkt, &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* if there are entries, pack the enviro entries */ for (j = 0; j < count; j++) { rc = PMIx_Data_pack(NULL, bkt, (void *) &app->env[j], 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } /* pack the cwd */ rc = PMIx_Data_pack(NULL, bkt, &app->cwd, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the flags */ rc = PMIx_Data_pack(NULL, bkt, &app->flags, 1, PMIX_INT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack attributes */ count = 0; PMIX_LIST_FOREACH(kv, &app->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { ++count; } } rc = PMIx_Data_pack(NULL, bkt, &count, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < count) { PMIX_LIST_FOREACH(kv, &app->attributes, prte_attribute_t) { if (PRTE_ATTR_GLOBAL == kv->local) { rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->key, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, (void *) &kv->data, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } } } return PRTE_SUCCESS; } /* * JOB_MAP * NOTE: There is no obvious reason to include all the node information when * sending a map */ int prte_map_pack(pmix_data_buffer_t *bkt, struct prte_job_map_t *mp) { pmix_status_t rc; prte_job_map_t *map = (prte_job_map_t *) mp; /* pack the requested mapper */ rc = PMIx_Data_pack(NULL, bkt, &map->req_mapper, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the last mapper */ rc = PMIx_Data_pack(NULL, bkt, &map->last_mapper, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the policies */ rc = PMIx_Data_pack(NULL, bkt, &map->mapping, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, &map->ranking, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } rc = PMIx_Data_pack(NULL, bkt, &map->binding, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } /* pack the number of nodes involved in the job */ rc = PMIx_Data_pack(NULL, bkt, &map->num_nodes, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/data_type_support/prte_dt_unpacking_fns.c0000664000175000017500000005072115203542132026376 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "types.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/runtime/prte_globals.h" /* * JOB * NOTE: We do not pack all of the job object's fields as many of them have no * value in sending them to another location. The only purpose in packing and * sending a job object is to communicate the data required to dynamically * spawn another job - so we only pack that limited set of required data. * Therefore, only unpack what was packed */ int prte_job_unpack(pmix_data_buffer_t *bkt, prte_job_t **job) { int rc; int32_t k, n, count, bookmark; prte_job_t *jptr; prte_app_idx_t j; prte_attribute_t *kv; char *tmp; prte_info_item_t *val; pmix_info_t pval; pmix_list_t *cache; /* create the prte_job_t object */ jptr = PMIX_NEW(prte_job_t); if (NULL == jptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* unpack the nspace */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->nspace, &n, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* unpack the flags */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->flags, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* unpack the attributes */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { kv = PMIX_NEW(prte_attribute_t); n = 1; rc = PMIx_Data_unpack(NULL, bkt, &kv->key, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } rc = PMIx_Data_unpack(NULL, bkt, &kv->data, &n, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } kv->local = PRTE_ATTR_GLOBAL; // obviously not a local value pmix_list_append(&jptr->attributes, &kv->super); } /* unpack any job info */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } if (0 < count) { cache = PMIX_NEW(pmix_list_t); prte_set_attribute(&jptr->attributes, PRTE_JOB_INFO_CACHE, PRTE_ATTR_LOCAL, (void *) cache, PMIX_POINTER); for (k = 0; k < count; k++) { n = 1; rc = PMIx_Data_unpack(NULL, bkt, &pval, &n, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } val = PMIX_NEW(prte_info_item_t); PMIX_INFO_XFER(&val->info, &pval); PMIX_INFO_DESTRUCT(&pval); pmix_list_append(cache, &val->super); } } /* unpack the personality */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { n = 1; rc = PMIx_Data_unpack(NULL, bkt, &tmp, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } PMIx_Argv_append_nosize(&jptr->personality, tmp); free(tmp); } /* unpack the num apps */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->num_apps, &n, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* if there are apps, unpack them */ if (0 < jptr->num_apps) { prte_app_context_t *app; for (j = 0; j < jptr->num_apps; j++) { n = 1; rc = prte_app_unpack(bkt, &app); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } pmix_pointer_array_add(jptr->apps, app); } } /* unpack num procs and offset */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->num_procs, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->offset, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } if (0 < jptr->num_procs) { prte_proc_t *proc; for (j = 0; j < jptr->num_procs; j++) { rc = prte_proc_unpack(bkt, &proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } pmix_pointer_array_add(jptr->procs, proc); } } /* unpack stdin target */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->stdin_target, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* unpack the total slots allocated to the job */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->total_slots_alloc, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* if the map is NULL, then we didn't pack it as there was * nothing to pack. Instead, we packed a flag to indicate whether or not * the map is included */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &j, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } if (0 < j) { /* unpack the map */ n = 1; rc = prte_map_unpack(bkt, &(jptr->map)); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } } /* unpack the bookmark */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &bookmark, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } if (0 <= bookmark) { /* retrieve it */ jptr->bookmark = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, bookmark); } /* unpack the job state */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->state, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } /* unpack the launcher ID */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &jptr->launcher, &n, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(jptr); return prte_pmix_convert_status(rc); } *job = jptr; return PRTE_SUCCESS; } /* * NODE */ int prte_node_unpack(pmix_data_buffer_t *bkt, prte_node_t **nd) { pmix_status_t rc; int32_t n, k, count; prte_node_t *node; uint8_t flag; prte_attribute_t *kv; /* create the node object */ node = PMIX_NEW(prte_node_t); if (NULL == node) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* unpack the node name */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &node->name, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); return prte_pmix_convert_status(rc); } /* unpack the number of procs on the node */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &node->num_procs, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); return prte_pmix_convert_status(rc); } /* unpack whether we are oversubscribed */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &flag, &n, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); return prte_pmix_convert_status(rc); } if (flag) { PRTE_FLAG_SET(node, PRTE_NODE_FLAG_OVERSUBSCRIBED); } /* unpack the state */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &node->state, &n, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); return prte_pmix_convert_status(rc); } /* unpack the attributes */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { kv = PMIX_NEW(prte_attribute_t); n = 1; rc = PMIx_Data_unpack(NULL, bkt, &kv->key, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } rc = PMIx_Data_unpack(NULL, bkt, &kv->data, &n, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(node); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } kv->local = PRTE_ATTR_GLOBAL; // obviously not a local value pmix_list_append(&node->attributes, &kv->super); } *nd = node; return PRTE_SUCCESS; } /* * PROC */ int prte_proc_unpack(pmix_data_buffer_t *bkt, prte_proc_t **pc) { pmix_status_t rc; int32_t n, count, k; prte_attribute_t *kv; ; prte_proc_t *proc; /* create the prte_proc_t object */ proc = PMIX_NEW(prte_proc_t); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* unpack the name */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->name, &n, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the node it is on */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->parent, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the local rank */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->local_rank, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the node rank */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->node_rank, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the state */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->state, &n, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the app context index */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->app_idx, &n, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the app_rank */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->app_rank, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the cpuset */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &proc->cpuset, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } /* unpack the attributes */ rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { kv = PMIX_NEW(prte_attribute_t); n = 1; rc = PMIx_Data_unpack(NULL, bkt, &kv->key, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } rc = PMIx_Data_unpack(NULL, bkt, &kv->data, &n, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(proc); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } kv->local = PRTE_ATTR_GLOBAL; // obviously not a local value pmix_list_append(&proc->attributes, &kv->super); } *pc = proc; return PRTE_SUCCESS; } /* * APP_CONTEXT */ int prte_app_unpack(pmix_data_buffer_t *bkt, prte_app_context_t **ap) { int rc; prte_app_context_t *app; int32_t n, count, k; prte_attribute_t *kv; char *tmp; /* create the app_context object */ app = PMIX_NEW(prte_app_context_t); if (NULL == app) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* get the app index number */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &app->idx, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* unpack the application name */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &app->app, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* get the number of processes */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &app->num_procs, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* get the first rank for this app */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &app->first_rank, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* get the number of argv strings that were packed */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { n = 1; rc = PMIx_Data_unpack(NULL, bkt, &tmp, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } PMIx_Argv_append_nosize(&app->argv, tmp); free(tmp); } /* get the number of env strings */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { n = 1; rc = PMIx_Data_unpack(NULL, bkt, &tmp, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } PMIx_Argv_append_nosize(&app->env, tmp); free(tmp); } /* unpack the cwd */ rc = PMIx_Data_unpack(NULL, bkt, &app->cwd, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* get the flags */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &app->flags, &n, PMIX_INT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } /* unpack the attributes */ rc = PMIx_Data_unpack(NULL, bkt, &count, &n, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); return prte_pmix_convert_status(rc); } for (k = 0; k < count; k++) { kv = PMIX_NEW(prte_attribute_t); n = 1; rc = PMIx_Data_unpack(NULL, bkt, &kv->key, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } rc = PMIx_Data_unpack(NULL, bkt, &kv->data, &n, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(app); PMIX_RELEASE(kv); return prte_pmix_convert_status(rc); } kv->local = PRTE_ATTR_GLOBAL; // obviously not a local value pmix_list_append(&app->attributes, &kv->super); } *ap = app; return PRTE_SUCCESS; } /* * JOB_MAP * NOTE: There is no obvious reason to include all the node information when * sending a map - hence, we do not pack that field, so don't unpack it here */ int prte_map_unpack(pmix_data_buffer_t *bkt, struct prte_job_map_t **mp) { int rc; int32_t n; prte_job_map_t *map; /* create the prte_rmaps_base_map_t object */ map = PMIX_NEW(prte_job_map_t); if (NULL == map) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* unpack the requested mapper */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->req_mapper, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } /* unpack the last mapper */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->last_mapper, &n, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } /* unpack the policies */ n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->mapping, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->ranking, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->binding, &n, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } /* unpack the number of nodes involved in the job */ n = 1; n = 1; rc = PMIx_Data_unpack(NULL, bkt, &map->num_nodes, &n, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(map); return prte_pmix_convert_status(rc); } *mp = map; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/prte_locks.c0000664000175000017500000000316315203542132020425 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #include "src/runtime/prte_locks.h" /* for everyone */ pmix_mutex_t prte_finalize_lock = PMIX_MUTEX_STATIC_INIT; /* for HNPs */ pmix_mutex_t prte_abort_inprogress_lock = PMIX_MUTEX_STATIC_INIT; pmix_mutex_t prte_jobs_complete_lock = PMIX_MUTEX_STATIC_INIT; pmix_mutex_t prte_quit_lock = PMIX_MUTEX_STATIC_INIT; pmix_lock_t prte_init_lock = PMIX_LOCK_STATIC_INIT; int prte_locks_init(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/runtime.h0000664000175000017500000000515115203542132017747 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Interface into the PRTE Run Time Environment */ #ifndef PRTE_RUNTIME_H #define PRTE_RUNTIME_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #include "src/util/proc_info.h" BEGIN_C_DECLS /** version string of ompi */ PRTE_EXPORT extern const char prte_version_string[]; /** * Whether PRTE is initialized or we are in prte_finalize */ PRTE_EXPORT extern bool prte_initialized; PRTE_EXPORT extern bool prte_finalizing; PRTE_EXPORT extern int prte_debug_output; PRTE_EXPORT extern bool prte_debug_flag; PRTE_EXPORT extern int prte_cache_line_size; /** * Initialize the Open Run Time Environment * * Initlize the Open Run Time Environment, including process * control, malloc debugging and threads, and out of band messaging. * This function should be called exactly once. This function should * be called by every application using the RTE interface, including * MPI applications and mpirun. * * @param pargc Pointer to the number of arguments in the pargv array * @param pargv The list of arguments. * @param flags Whether we are PRTE tool or not */ PRTE_EXPORT int prte_init(int *pargc, char ***pargv, prte_proc_type_t flags); PRTE_EXPORT int prte_init_util(prte_proc_type_t flags); PRTE_EXPORT int prte_init_minimum(void); /** * Initialize parameters for PRTE. * * @retval PRTE_SUCCESS Upon success. * @retval PRTE_ERROR Upon failure. */ PRTE_EXPORT int prte_register_params(void); /** * Finalize the Open run time environment. Any function calling \code * prte_init should call \code prte_finalize. * */ PRTE_EXPORT int prte_finalize(void); END_C_DECLS #endif /* RUNTIME_H */ prrte-4.1.1~rc2/src/runtime/prte_init.c0000664000175000017500000006032515203542132020260 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_STRING_H #include #endif #include "src/util/error.h" #include "src/util/error_strings.h" #include "src/util/pmix_keyval_parse.h" #include "src/util/malloc.h" #include "src/util/name_fns.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_os_path.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/util/stacktrace.h" #include "src/util/sys_limits.h" #include "src/hwloc/hwloc-internal.h" #include "src/prted/pmix/pmix_server.h" #include "src/threads/pmix_threads.h" #include "src/include/prte_frameworks.h" #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/base/pmix_mca_base_vari.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/filem/base/base.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/base/base.h" #include "src/mca/pmdl/base/base.h" #include "src/mca/prtebacktrace/base/base.h" #include "src/mca/prteinstalldirs/base/base.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/schizo/base/base.h" #include "src/mca/state/base/base.h" #include "src/runtime/pmix_init_util.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/runtime.h" #include "src/runtime/runtime_internals.h" /* * Whether we have completed prte_init or we are in prte_finalize */ bool prte_initialized = false; bool prte_finalizing = false; bool prte_debug_flag = false; int prte_debug_verbosity = -1; char *prte_prohibited_session_dirs = NULL; bool prte_create_session_dirs = true; prte_event_base_t *prte_event_base = {0}; bool prte_event_base_active = true; bool prte_proc_is_bound = false; int prte_progress_thread_debug = -1; hwloc_cpuset_t prte_proc_applied_binding = NULL; int prte_cache_line_size = 128; pmix_proc_t prte_name_wildcard = {{0}, PMIX_RANK_WILDCARD}; pmix_proc_t prte_name_invalid = {{0}, PMIX_RANK_INVALID}; pmix_nspace_t prte_nspace_wildcard = {0}; static bool util_initialized = false; static bool min_initialized = false; const char prte_version_string[] = PRTE_IDENT_STRING; static bool check_exist(char *path) { struct stat buf; /* coverity[TOCTOU] */ if (0 == stat(path, &buf)) { /* exists */ return true; } return false; } static void print_error(unsigned major, unsigned minor, unsigned release, unsigned reqmajor, unsigned reqminor, unsigned reqrelease, char *minmax, char *limit) { fprintf(stderr, "************************************************\n"); fprintf(stderr, "We have detected that the runtime version\n"); fprintf(stderr, "of the PMIx library we were given violates\n"); fprintf(stderr, "the %s version requirement:\n\n", minmax); fprintf(stderr, " Runtime: 0x%x%02x%02x\n", major, minor, release); fprintf(stderr, " %s: 0x%x%02x%02x\n\n", limit, reqmajor, reqminor, reqrelease); fprintf(stderr, "Please update your LD_LIBRARY_PATH to point\n"); fprintf(stderr, "us to the same PMIx version used to build PRRTE.\n"); fprintf(stderr, "************************************************\n"); } static bool check_range(unsigned major, unsigned minor, unsigned release, unsigned limmajor, unsigned limminor, unsigned limrelease) { if (major > limmajor) { // we are good - nothing more to check return true; } else if (major < limmajor) { return false; } // checking within the major release series if (minor > limminor) { // we are good - nothing more to check return true; } else if (minor < limminor) { return false; } // checking within the minor release series if (release < limrelease) { return false; } return true; } int prte_init_minimum(void) { int ret, n; char *path = NULL; char *evar, **prefixes; const char *rvers; char token[100]; unsigned int major, minor, release; unsigned int reqmajor, reqminor, reqrelease; unsigned int maxmajor, maxminor, maxrelease; if (min_initialized) { return PRTE_SUCCESS; } min_initialized = true; /* check to see if the version of PMIx we were given in the * library path matches the version we were built against. * Because we are using PMIx internals, we cannot support * cross version operations from inside of PRRTE. */ rvers = PMIx_Get_version(); ret = sscanf(rvers, "%s %u.%u.%u", token, &major, &minor, &release); ret = sscanf(PRTE_PMIX_MIN_VERSION_STRING, "%u.%u.%u", &reqmajor, &reqminor, &reqrelease); ret = sscanf(PRTE_PMIX_MAX_VERSION_STRING, "%u.%u.%u", &maxmajor, &maxminor, &maxrelease); /* check the version triplet agains the min values * specified in VERSION */ if (!check_range(major, minor, release, reqmajor, reqminor, reqrelease)) { print_error(major, minor, release, reqmajor, reqminor, reqrelease, "minimum", "Minimum"); return PRTE_ERR_SILENT; } /* check the version triplet agains the max values * specified in VERSION */ if (check_range(major, minor, release, maxmajor, maxminor, maxrelease)) { print_error(major, minor, release, maxmajor, maxminor, maxrelease, "maximum", "Maximum"); return PRTE_ERR_SILENT; } /* Protect against the envar version of the Slurm * custom args MCA param. This is an unfortunate * hack that hopefully will eventually go away. * See both of the following for detailed * explanations and discussion: * * https://github.com/openpmix/prrte/issues/1974 * https://github.com/open-mpi/ompi/issues/12471 * * Orgs/users wanting to add custom args to the * internal "srun" command used to spawn the * PRRTE daemons must do so via the default MCA * param files (system or user), or via the * prterun (or its proxy) cmd line */ unsetenv("PRTE_MCA_plm_slurm_args"); unsetenv("OMPI_MCA_plm_slurm_args"); /* carry across the toolname */ pmix_tool_basename = prte_tool_basename; // publish MCA prefixes prefixes = NULL; for (n=0; NULL != prte_framework_names[n]; n++) { if (0 == strcmp("common", prte_framework_names[n])) { continue; } PMIx_Argv_append_nosize(&prefixes, prte_framework_names[n]); } evar = PMIx_Argv_join(prefixes, ','); pmix_setenv("PRTE_MCA_PREFIXES", evar, true, &environ); free(evar); PMIx_Argv_free(prefixes); /* initialize install dirs code */ ret = pmix_mca_base_framework_open(&prte_prteinstalldirs_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT); if (PRTE_SUCCESS != ret) { fprintf(stderr, "prte_prteinstalldirs_base_open() failed -- process will likely abort (%s:%d, " "returned %d instead of PRTE_SUCCESS)\n", __FILE__, __LINE__, ret); return ret; } /* initialize the MCA infrastructure */ if (check_exist(prte_install_dirs.pmixlibdir)) { pmix_asprintf(&path, "prte@%s", prte_install_dirs.pmixlibdir); } ret = pmix_init_util(NULL, 0, path); if (NULL != path) { free(path); } if (PMIX_SUCCESS != ret) { return prte_pmix_convert_status(ret); } pmix_show_help_add_data("prte", prte_show_help_data); /* keyval lex-based parser */ /* Setup the parameter system */ if (PRTE_SUCCESS != (ret = pmix_mca_base_var_init())) { return ret; } /* Register all global MCA Params */ if (PRTE_SUCCESS != (ret = prte_register_params())) { if (PRTE_ERR_SILENT != ret) { pmix_show_help("help-prte-runtime", "prte_init:startup:internal-failure", true, "prte register params", PRTE_ERROR_NAME(ret), ret); } return 1; } /* pre-load any default mca param files */ prte_preload_default_mca_params(); return PRTE_SUCCESS; } int prte_init_util(prte_proc_type_t flags) { int ret; char *error = NULL; if (util_initialized) { return PRTE_SUCCESS; } util_initialized = true; ret = prte_init_minimum(); if (PRTE_SUCCESS != ret) { return ret; } /* ensure we know the type of proc for when we finalize */ prte_process_info.proc_type = flags; /* initialize the memory allocator */ prte_malloc_init(); /* initialize the output system */ pmix_output_init(); /* set the nodename so anyone who needs it has it - this * must come AFTER we initialize the installdirs */ prte_setup_hostname(); /* pretty-print stack handlers */ if (PRTE_SUCCESS != (ret = prte_util_register_stackhandlers())) { error = "prte_util_register_stackhandlers"; goto error; } /* set system resource limits - internally protected against * doing so twice in cases where the launch agent did it for us */ if (PRTE_SUCCESS != (ret = prte_util_init_sys_limits(&error))) { pmix_show_help("help-prte-runtime.txt", "prte_init:syslimit", false, error); return PRTE_ERR_SILENT; } ret = pmix_mca_base_framework_open(&prte_prtebacktrace_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT); if (PRTE_SUCCESS != ret) { error = "prte_backtrace_base_open"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret) { pmix_show_help("help-prte-runtime", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } int prte_init(int *pargc, char ***pargv, prte_proc_type_t flags) { int ret; char *error = NULL; PMIX_ACQUIRE_THREAD(&prte_init_lock); if (prte_initialized) { PMIX_RELEASE_THREAD(&prte_init_lock); return PRTE_SUCCESS; } PMIX_RELEASE_THREAD(&prte_init_lock); ret = prte_init_util(flags); if (PRTE_SUCCESS != ret) { return ret; } /* * Initialize the event library */ if (PRTE_SUCCESS != (ret = prte_event_base_open())) { error = "prte_event_base_open"; goto error; } /* setup the locks */ if (PRTE_SUCCESS != (ret = prte_locks_init())) { error = "prte_locks_init"; goto error; } /* Ensure the rest of the process info structure is initialized */ if (PRTE_SUCCESS != (ret = prte_proc_info())) { error = "prte_proc_info"; goto error; } if (PRTE_SUCCESS != (ret = prte_hwloc_base_register())) { error = "prte_hwloc_base_register"; goto error; } /* let the pmix server register params */ pmix_server_register_params(); /* open hwloc */ prte_hwloc_base_open(); /* setup the global job and node arrays */ prte_job_data = PMIX_NEW(pmix_pointer_array_t); ret = pmix_pointer_array_init(prte_job_data, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "setup job array"; goto error; } prte_node_pool = PMIX_NEW(pmix_pointer_array_t); ret = pmix_pointer_array_init(prte_node_pool, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "setup node array"; goto error; } prte_sessions = PMIX_NEW(pmix_pointer_array_t); ret = pmix_pointer_array_init(prte_sessions, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "setup session array"; goto error; } /* Create a session object for the DVM and let it point to the node pool */ prte_default_session = PMIX_NEW(prte_session_t); prte_default_session->session_id = UINT32_MAX; PMIX_RELEASE(prte_default_session->nodes); prte_default_session->nodes = prte_node_pool; ret = prte_set_session_object(prte_default_session); if (PRTE_SUCCESS != ret) { error = "set session object"; goto error; } prte_node_topologies = PMIX_NEW(pmix_pointer_array_t); ret = pmix_pointer_array_init(prte_node_topologies, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "setup node topologies array"; goto error; } /* open the SCHIZO framework as everyone needs it, and the * ess will use it to help select its component */ ret = pmix_mca_base_framework_open(&prte_schizo_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "prte_schizo_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_schizo_base_select())) { error = "prte_schizo_base_select"; goto error; } /* open the ESS and select the correct module for this environment */ ret = pmix_mca_base_framework_open(&prte_ess_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); error = "prte_ess_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_ess_base_select())) { error = "prte_ess_base_select"; goto error; } /* initialize the RTE for this environment */ if (PRTE_SUCCESS != (ret = prte_ess.init(*pargc, *pargv))) { error = "prte_ess_init"; goto error; } /* initialize the cache */ prte_cache = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(prte_cache, 1, INT_MAX, 1); /* All done */ PMIX_ACQUIRE_THREAD(&prte_init_lock); prte_initialized = true; PMIX_RELEASE_THREAD(&prte_init_lock); return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret) { pmix_show_help("help-prte-runtime", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } static bool check_pmix_overlap(char *var, char *value, bool overwrite) { char *tmp; if (0 == strncmp(var, "dl_", 3)) { pmix_asprintf(&tmp, "PMIX_MCA_pdl_%s", &var[3]); setenv(tmp, value, overwrite); free(tmp); return true; } else if (0 == strncmp(var, "oob_", 4) && NULL == strstr(var, "verbose")) { pmix_asprintf(&tmp, "PMIX_MCA_ptl_%s", &var[4]); setenv(tmp, value, overwrite); free(tmp); return true; } else if (0 == strncmp(var, "hwloc_", 6)) { pmix_asprintf(&tmp, "PMIX_MCA_%s", var); setenv(tmp, value, overwrite); free(tmp); return true; } else if (0 == strncmp(var, "if_", 3)) { // need to convert if to pif pmix_asprintf(&tmp, "PMIX_MCA_pif_%s", &var[3]); setenv(tmp, value, overwrite); free(tmp); return true; } else if (0 == strncmp(var, "mca_", 4)) { pmix_asprintf(&tmp, "PMIX_MCA_%s", var); setenv(tmp, value, overwrite); free(tmp); return true; } return false; } int prte_preload_default_mca_params(void) { char *file, *home, *tmp, **paths = NULL; pmix_list_t params, params2, pfinal; pmix_mca_base_var_file_value_t *fv, *fv2, *fvnext, *fvnext2; bool match; int i, rc; home = (char*)pmix_home_directory(-1); PMIX_CONSTRUCT(¶ms, pmix_list_t); PMIX_CONSTRUCT(¶ms2, pmix_list_t); PMIX_CONSTRUCT(&pfinal, pmix_list_t); if (NULL == prte_param_files) { /* start with the system-level defaults */ file = pmix_os_path(false, prte_install_dirs.sysconfdir, "prte-mca-params.conf", NULL); rc = pmix_mca_base_parse_paramfile(file, ¶ms); free(file); if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_FOUND != rc) { // it is okay if the file isn't found return rc; } /* now get the user-level defaults */ file = pmix_os_path(false, home, ".prte", "mca-params.conf", NULL); rc = pmix_mca_base_parse_paramfile(file, ¶ms2); free(file); if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_FOUND != rc) { // it is okay if the file isn't found return rc; } } else { // split the string on commas paths = PMIx_Argv_split(prte_param_files, ','); // process each path for (i=0; NULL != paths[i]; i++) { rc = pmix_mca_base_parse_paramfile(paths[i], ¶ms); if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_FOUND != rc) { // it is okay if the file isn't found PMIx_Argv_free(paths); return rc; } } PMIx_Argv_free(paths); } /* cross-check the lists, keeping the params2 entries over any * matching params entries as they overwrite the system ones */ PMIX_LIST_FOREACH_SAFE(fv, fvnext, ¶ms, pmix_mca_base_var_file_value_t) { match = false; PMIX_LIST_FOREACH_SAFE(fv2, fvnext2, ¶ms2, pmix_mca_base_var_file_value_t) { /* do we have a match? */ if (0 == strcmp(fv->mbvfv_var, fv2->mbvfv_var)) { /* transfer the user-level default to the final list */ pmix_list_remove_item(¶ms2, &fv2->super); pmix_list_append(&pfinal, &fv2->super); /* remove and release the system-level duplicate */ pmix_list_remove_item(¶ms, &fv->super); PMIX_RELEASE(fv); match = true; break; } } if (!match) { /* transfer the system-level default to the final list */ pmix_list_remove_item(¶ms, &fv->super); pmix_list_append(&pfinal, &fv->super); } } /* transfer any remaining use-level defaults to the final list * as they had no matches */ while (NULL != (fv2 = (pmix_mca_base_var_file_value_t*)pmix_list_remove_first(¶ms2))) { pmix_list_append(&pfinal, &fv2->super); } PMIX_LIST_DESTRUCT(¶ms); PMIX_LIST_DESTRUCT(¶ms2); // process any override params PMIX_CONSTRUCT(¶ms, pmix_list_t); if (NULL != prte_override_param_file) { // process the file rc = pmix_mca_base_parse_paramfile(prte_override_param_file, ¶ms); if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_FOUND != rc) { // it is okay if the file isn't found PMIX_LIST_DESTRUCT(¶ms); PMIX_LIST_DESTRUCT(&pfinal); return rc; } if (0 < pmix_list_get_size(¶ms)) { // check the params against any given PMIX_LIST_FOREACH(fv, &pfinal, pmix_mca_base_var_file_value_t) { PMIX_LIST_FOREACH(fv2, ¶ms, pmix_mca_base_var_file_value_t) { if (0 == strcmp(fv->mbvfv_var, fv2->mbvfv_var)) { if (!prte_suppress_override_warning) { pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, fv->mbvfv_var); } free(fv->mbvfv_var); fv->mbvfv_var = strdup(fv2->mbvfv_var); free(fv->mbvfv_value); fv->mbvfv_value = strdup(fv2->mbvfv_value); break; } } } // now overwrite any envars PMIX_LIST_FOREACH(fv, ¶ms, pmix_mca_base_var_file_value_t) { if (pmix_pmdl_base_check_pmix_param(fv->mbvfv_var)) { pmix_asprintf(&tmp, "PMIX_MCA_%s", fv->mbvfv_var); if (!prte_suppress_override_warning && NULL != getenv(tmp)) { pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, tmp); } // set it, and overwrite if they already // have a value in our environment setenv(tmp, fv->mbvfv_value, true); free(tmp); } else { pmix_asprintf(&tmp, "PRTE_MCA_%s", fv->mbvfv_var); if (!prte_suppress_override_warning && NULL != getenv(tmp)) { pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, tmp); } // set it, and overwrite if they already // have a value in our environment setenv(tmp, fv->mbvfv_value, true); free(tmp); // if this relates to the DL, OOB, HWLOC, or IF, // or mca frameworks, then we also need to set // the equivalent PMIx value if (check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value, true) && !prte_suppress_override_warning) { pmix_show_help("help-pmix-mca-var.txt", "overridden-param-set", true, fv->mbvfv_var); } } } } PMIX_LIST_DESTRUCT(¶ms); } /* now process the final list - but do not overwrite if the * user already has the param in our environment as their * environment settings override all defaults */ PMIX_LIST_FOREACH(fv, &pfinal, pmix_mca_base_var_file_value_t) { if (pmix_pmdl_base_check_pmix_param(fv->mbvfv_var)) { pmix_asprintf(&tmp, "PMIX_MCA_%s", fv->mbvfv_var); // set it, but don't overwrite if they already // have a value in our environment setenv(tmp, fv->mbvfv_value, false); free(tmp); } else { pmix_asprintf(&tmp, "PRTE_MCA_%s", fv->mbvfv_var); // set it, but don't overwrite if they already // have a value in our environment setenv(tmp, fv->mbvfv_value, false); free(tmp); // if this relates to the DL, OOB, HWLOC, or IF, // or mca frameworks, then we also need to set // the equivalent PMIx value check_pmix_overlap(fv->mbvfv_var, fv->mbvfv_value, false); } } PMIX_LIST_DESTRUCT(&pfinal); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/prte_progress_threads.h0000664000175000017500000000440615203542132022676 0ustar alastairalastair/* * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PROGRESS_THREADS_H #define PRTE_PROGRESS_THREADS_H #include "prte_config.h" #include "src/event/event-internal.h" /** * Initialize a progress thread name; if a progress thread is not * already associated with that name, start a progress thread. * * If you have general events that need to run in *a* progress thread * (but not necessarily a your own, dedicated progress thread), pass * NULL the "name" argument to the prte_progress_thead_init() function * to glom on to the general PRTE-wide progress thread. * * If a name is passed that was already used in a prior call to * prte_progress_thread_init(), the event base associated with that * already-running progress thread will be returned (i.e., no new * progress thread will be started). */ PRTE_EXPORT prte_event_base_t *prte_progress_thread_init(const char *name); /** * Finalize a progress thread name (reference counted). * * Once this function is invoked as many times as * prte_progress_thread_init() was invoked on this name (or NULL), the * progress function is shut down and the event base associated with * it is destroyed. * * Will return PRTE_ERR_NOT_FOUND if the progress thread name does not * exist; PRTE_SUCCESS otherwise. */ PRTE_EXPORT int prte_progress_thread_finalize(const char *name); /** * Temporarily pause the progress thread associated with this name. * * This function does not destroy the event base associated with this * progress thread name, but it does stop processing all events on * that event base until prte_progress_thread_resume() is invoked on * that name. * * Will return PRTE_ERR_NOT_FOUND if the progress thread name does not * exist; PRTE_SUCCESS otherwise. */ PRTE_EXPORT int prte_progress_thread_pause(const char *name); /** * Restart a previously-paused progress thread associated with this * name. * * Will return PRTE_ERR_NOT_FOUND if the progress thread name does not * exist; PRTE_SUCCESS otherwise. */ PRTE_EXPORT int prte_progress_thread_resume(const char *name); #endif prrte-4.1.1~rc2/src/runtime/prte_globals.c0000664000175000017500000007205015203542132020736 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_hash_table.h" #include "src/class/pmix_pointer_array.h" #include "src/class/pmix_value_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/threads/pmix_threads.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/rmaps.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/util/pmix_argv.h" #include "src/util/name_fns.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/runtime/prte_globals.h" #include "src/runtime/runtime.h" #include "src/runtime/runtime_internals.h" /* State Machine */ pmix_list_t prte_job_states = PMIX_LIST_STATIC_INIT; pmix_list_t prte_proc_states = PMIX_LIST_STATIC_INIT; /* a clean output channel without prefix */ int prte_clean_output = -1; /* globals used by RTE */ bool prte_debug_daemons_file_flag = false; bool prte_leave_session_attached = false; char *prte_data_server_uri = NULL; char *prte_tool_basename = NULL; char *prte_tool_actual = NULL; bool prte_dvm_ready = false; pmix_pointer_array_t *prte_cache = NULL; bool prte_persistent = true; bool prte_allow_run_as_root = false; bool prte_fwd_environment = false; bool prte_show_launch_progress = false; bool prte_bootstrap_setup = false; bool prte_xml_output = false; /* PRTE OOB port flags */ bool prte_static_ports = false; char *prte_oob_static_ports = NULL; bool prte_keep_fqdn_hostnames = false; bool prte_have_fqdn_allocation = false; bool prte_show_resolved_nodenames = false; bool prte_do_not_resolve = false; int prte_hostname_cutoff = 1000; pmix_rank_t prted_debug_failure = PMIX_RANK_INVALID; int prted_debug_failure_delay = -1; bool prte_never_launched = false; bool prte_devel_level_output = false; bool prte_display_topo_with_map = false; char **prte_launch_environ = NULL; bool prte_hnp_is_allocated = false; bool prte_allocation_required = false; bool prte_managed_allocation = false; char *prte_set_slots = NULL; bool prte_set_slots_override = false; bool prte_nidmap_communicated = false; bool prte_node_info_communicated = false; /* launch agents */ char *prte_launch_agent = NULL; char **prted_cmd_line = NULL; /* exit flags */ int prte_exit_status = 0; bool prte_abnormal_term_ordered = false; bool prte_routing_is_enabled = true; bool prte_dvm_abort_ordered = false; bool prte_prteds_term_ordered = false; bool prte_allowed_exit_without_sync = false; int prte_timeout_usec_per_proc = -1; float prte_max_timeout = -1.0; prte_timer_t *prte_mpiexec_timeout = NULL; int prte_stack_trace_wait_timeout = 30; /* global arrays for data storage */ pmix_pointer_array_t *prte_sessions = NULL; pmix_pointer_array_t *prte_job_data = NULL; pmix_pointer_array_t *prte_node_pool = NULL; pmix_pointer_array_t *prte_node_topologies = NULL; pmix_pointer_array_t *prte_local_children = NULL; pmix_rank_t prte_total_procs = 0; char *prte_base_compute_node_sig = NULL; bool prte_homo_nodes = false; /* IOF controls */ /* generate new xterm windows to display output from specified ranks */ char *prte_xterm = NULL; /* report launch progress */ bool prte_report_launch_progress = false; /* allocation specification */ char *prte_default_hostfile = NULL; bool prte_default_hostfile_given = false; int prte_num_allocated_nodes = 0; char *prte_default_dash_host = NULL; prte_session_t *prte_default_session = NULL; /* tool communication controls */ bool prte_report_events = false; char *prte_report_events_uri = NULL; /* report bindings */ bool prte_report_bindings = false; /* exit status reporting */ bool prte_report_child_jobs_separately = false; struct timeval prte_child_time_to_exit = {0}; /* length of stat history to keep */ int prte_stat_history_size = -1; /* maximum size of virtual machine - used to subdivide allocation */ int prte_max_vm_size = -1; int prte_debug_output = -1; bool prte_debug_daemons_flag = false; char *prte_job_ident = NULL; bool prte_execute_quiet = false; bool prte_report_silent_errors = false; bool prte_hwloc_shmem_available = false; /* See comment in src/tools/prun/debuggers.c about this MCA param */ bool prte_in_parallel_debugger = false; char *prte_daemon_cores = NULL; int prte_dt_init(void) { /* set default output */ prte_debug_output = pmix_output_open(NULL); /* open up the verbose output for PRTE debugging */ if (prte_debug_flag || 0 < prte_debug_verbosity || (prte_debug_daemons_flag && (PRTE_PROC_IS_DAEMON || PRTE_PROC_IS_MASTER))) { if (0 < prte_debug_verbosity) { pmix_output_set_verbosity(prte_debug_output, prte_debug_verbosity); } else { pmix_output_set_verbosity(prte_debug_output, 1); } } return PRTE_SUCCESS; } prte_job_t *prte_get_job_data_object(const pmix_nspace_t job) { prte_job_t *jptr; int i; /* if the job data wasn't setup, we cannot provide the data */ if (NULL == prte_job_data) { return NULL; } /* if the nspace is invalid, then reject it */ if (PMIX_NSPACE_INVALID(job)) { return NULL; } for (i = 0; i < prte_job_data->size; i++) { if (NULL == (jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, i))) { continue; } if (PMIX_CHECK_NSPACE(jptr->nspace, job)) { return jptr; } } return NULL; } int prte_set_job_data_object(prte_job_t *jdata) { prte_job_t *jptr; int i, save = -1; /* if the job data wasn't setup, we cannot set the data */ if (NULL == prte_job_data) { return PRTE_ERROR; } /* if the nspace is invalid, then that's an error */ if (PMIX_NSPACE_INVALID(jdata->nspace)) { return PRTE_ERROR; } /* verify that we don't already have this object */ for (i = 0; i < prte_job_data->size; i++) { if (NULL == (jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, i))) { if (0 > save) { save = i; } continue; } if (PMIX_CHECK_NSPACE(jptr->nspace, jdata->nspace)) { return PRTE_EXISTS; } } if (-1 == save) { jdata->index = pmix_pointer_array_add(prte_job_data, jdata); } else { jdata->index = save; pmix_pointer_array_set_item(prte_job_data, save, jdata); } if (0 > jdata->index) { return PRTE_ERROR; } return PRTE_SUCCESS; } prte_session_t *prte_get_session_object(const uint32_t session_id) { prte_session_t *session; int i; /* if the job data wasn't setup, we cannot provide the data */ if (NULL == prte_sessions) { return NULL; } for (i = 0; i < prte_sessions->size; i++) { session = (prte_session_t *) pmix_pointer_array_get_item(prte_sessions, i); if (NULL == session) { continue; } if (session->session_id == session_id) { return session; } } return NULL; } prte_session_t *prte_get_session_object_from_id(const char *id) { prte_session_t *session; int i; /* if the job data wasn't setup, we cannot provide the data */ if (NULL == prte_sessions) { return NULL; } /* if the session ID is invalid, then reject it */ if (NULL == id) { return NULL; } for (i = 0; i < prte_sessions->size; i++) { session = (prte_session_t *) pmix_pointer_array_get_item(prte_sessions, i); if (NULL == session) { continue; } if (0 == strcasecmp(session->alloc_refid, id)) { return session; } } return NULL; } prte_session_t *prte_get_session_object_from_refid(const char *refid) { prte_session_t *session; int i; /* if the job data wasn't setup, we cannot provide the data */ if (NULL == prte_sessions) { return NULL; } /* if the session ID is invalid, then reject it */ if (NULL == refid) { return NULL; } for (i = 0; i < prte_sessions->size; i++) { session = (prte_session_t *) pmix_pointer_array_get_item(prte_sessions, i); if (NULL == session) { continue; } if (0 == strcasecmp(session->user_refid, refid)) { return session; } } return NULL; } int prte_set_session_object(prte_session_t *session) { prte_session_t *session_ptr; int i, save = -1; /* if the session data wasn't setup, we cannot set the data */ if (NULL == prte_sessions) { return PRTE_ERROR; } /* verify that we don't already have this object */ for (i = 0; i < prte_sessions->size; i++) { session_ptr = (prte_session_t *) pmix_pointer_array_get_item(prte_sessions, i); if (NULL == session_ptr) { if (0 > save) { save = i; } continue; } if (session_ptr->session_id == session->session_id) { return PRTE_EXISTS; } } if (-1 == save) { session->index = pmix_pointer_array_add(prte_sessions, session); if (0 > session->index) { return PRTE_ERROR; } } else { session->index = save; pmix_pointer_array_set_item(prte_sessions, save, session); } return PRTE_SUCCESS; } bool prte_sessions_related(prte_session_t *session1, prte_session_t *session2){ int n; prte_session_t *session_ptr; if(session1->session_id == session2->session_id){ return true; } for (n = 0; n < session1->children->size; n++){ session_ptr = (prte_session_t *) pmix_pointer_array_get_item(session1->children, n); if (NULL != session_ptr) { if (session_ptr->session_id == session2->session_id) { return true; } } } return false; } prte_proc_t *prte_get_proc_object(const pmix_proc_t *proc) { prte_job_t *jdata; prte_proc_t *proct; if (NULL == (jdata = prte_get_job_data_object(proc->nspace))) { return NULL; } proct = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank); return proct; } pmix_rank_t prte_get_proc_daemon_vpid(const pmix_proc_t *proc) { prte_job_t *jdata; prte_proc_t *proct; if (NULL == (jdata = prte_get_job_data_object(proc->nspace))) { return PMIX_RANK_INVALID; } if (NULL == (proct = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank))) { return PMIX_RANK_INVALID; } if (NULL == proct->node || NULL == proct->node->daemon) { return PMIX_RANK_INVALID; } return proct->node->daemon->name.rank; } char *prte_get_proc_hostname(const pmix_proc_t *proc) { prte_proc_t *proct; /* don't bother error logging any not-found situations * as the layer above us will have something to say * about it */ /* look it up on our arrays */ if (NULL == (proct = prte_get_proc_object(proc))) { return NULL; } if (NULL == proct->node || NULL == proct->node->name) { return NULL; } return proct->node->name; } prte_node_rank_t prte_get_proc_node_rank(const pmix_proc_t *proc) { prte_proc_t *proct; /* look it up on our arrays */ if (NULL == (proct = prte_get_proc_object(proc))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_NODE_RANK_INVALID; } return proct->node_rank; } prte_node_t* prte_node_match(pmix_list_t *nodes, const char *name) { int m, n; prte_node_t *nptr; char *nm; /* does the name refer to me? */ if (prte_check_host_is_local(name)) { nm = prte_process_info.nodename; } else { nm = (char*)name; } if (NULL != nodes) { PMIX_LIST_FOREACH(nptr, nodes, prte_node_t) { if (0 == strcmp(nptr->name, nm)) { return nptr; } if (NULL == nptr->aliases) { continue; } /* no choice but an exhaustive search - fortunately, these lists are short! */ for (m = 0; NULL != nptr->aliases[m]; m++) { if (0 == strcmp(name, nptr->aliases[m])) { /* this is the node! */ return nptr; } } } } else { /* check the node pool */ for (n=0; n < prte_node_pool->size; n++) { nptr = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, n); if (NULL == nptr) { continue; } if (0 == strcmp(nptr->name, nm)) { return nptr; } if (NULL == nptr->aliases) { continue; } /* no choice but an exhaustive search - fortunately, these lists are short! */ for (m = 0; NULL != nptr->aliases[m]; m++) { if (0 == strcmp(name, nptr->aliases[m])) { /* this is the node! */ return nptr; } } } } return NULL; } bool prte_nptr_match(prte_node_t *n1, prte_node_t *n2) { size_t i, m; /* start with the simple check */ if (0 == strcmp(n1->name, n2->name)) { return true; } if (NULL != n1->aliases) { for (i = 0; NULL != n1->aliases[i]; i++) { if (0 == strcmp(n1->aliases[i], n2->name)) { return true; } if (NULL != n2->aliases) { for (m = 0; NULL != n2->aliases[m]; m++) { if (0 == strcmp(n2->aliases[m], n1->name)) { return true; } if (0 == strcmp(n1->aliases[i], n2->aliases[m])) { return true; } } } } } return false; } bool prte_quickmatch(prte_node_t *nd, char *name) { int n; if (0 == strcmp(nd->name, name)) { return true; } if (prte_check_host_is_local(nd->name) && prte_check_host_is_local(name)) { return true; } if (NULL != nd->aliases) { for (n=0; NULL != nd->aliases[n]; n++) { if (0 == strcmp(nd->aliases[n], name)) { return true; } } } return false; } /* * CONSTRUCTORS, DESTRUCTORS, AND CLASS INSTANTIATIONS * FOR PRTE CLASSES */ static void prte_app_context_construct(prte_app_context_t *app_context) { app_context->job = NULL; app_context->idx = 0; app_context->app = NULL; app_context->num_procs = 0; PMIX_CONSTRUCT(&app_context->procs, pmix_pointer_array_t); pmix_pointer_array_init(&app_context->procs, 1, PRTE_GLOBAL_ARRAY_MAX_SIZE, 16); app_context->state = PRTE_APP_STATE_UNDEF; app_context->first_rank = 0; app_context->argv = NULL; app_context->env = NULL; app_context->cwd = NULL; app_context->flags = 0; PMIX_CONSTRUCT(&app_context->attributes, pmix_list_t); PMIX_CONSTRUCT(&app_context->cli, pmix_cli_result_t); } static void prte_app_context_destructor(prte_app_context_t *app_context) { int i; prte_proc_t *proc; if (NULL != app_context->app) { free(app_context->app); app_context->app = NULL; } for (i = 0; i < app_context->procs.size; i++) { if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&app_context->procs, i))) { PMIX_RELEASE(proc); } } PMIX_DESTRUCT(&app_context->procs); /* argv and env lists created by util/argv copy functions */ if (NULL != app_context->argv) { PMIx_Argv_free(app_context->argv); app_context->argv = NULL; } if (NULL != app_context->env) { PMIx_Argv_free(app_context->env); app_context->env = NULL; } if (NULL != app_context->cwd) { free(app_context->cwd); app_context->cwd = NULL; } PMIX_LIST_DESTRUCT(&app_context->attributes); PMIX_DESTRUCT(&app_context->cli); } PMIX_CLASS_INSTANCE(prte_app_context_t, pmix_object_t, prte_app_context_construct, prte_app_context_destructor); static void prte_job_construct(prte_job_t *job) { job->exit_code = 0; job->session = NULL; job->personality = NULL; job->schizo = NULL; PMIX_LOAD_NSPACE(job->nspace, NULL); job->session_dir = NULL; job->index = -1; job->offset = 0; job->apps = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(job->apps, 1, PRTE_GLOBAL_ARRAY_MAX_SIZE, 2); job->num_apps = 0; job->stdin_target = 0; job->total_slots_alloc = 0; job->num_procs = 0; job->procs = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(job->procs, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); job->map = NULL; job->bookmark = NULL; job->state = PRTE_JOB_STATE_UNDEF; job->num_mapped = 0; job->num_launched = 0; job->num_reported = 0; job->num_terminated = 0; job->num_daemons_reported = 0; job->num_ready_for_debug = 0; PMIX_LOAD_PROCID(&job->originator, NULL, PMIX_RANK_INVALID); job->num_local_procs = 0; job->flags = 0; PRTE_FLAG_SET(job, PRTE_JOB_FLAG_FORWARD_OUTPUT); PMIX_CONSTRUCT(&job->attributes, pmix_list_t); PMIX_DATA_BUFFER_CONSTRUCT(&job->launch_msg); PMIX_CONSTRUCT(&job->children, pmix_list_t); PMIX_LOAD_NSPACE(job->launcher, NULL); job->ntraces = 0; job->traces = NULL; PMIX_CONSTRUCT(&job->cli, pmix_cli_result_t); } static void prte_job_destruct(prte_job_t *job) { prte_proc_t *proc; prte_app_context_t *app; int n; prte_timer_t *evtimer; pmix_list_t *cache = NULL; if (NULL == job) { /* probably just a race condition - just return */ return; } if (NULL != job->personality) { PMIx_Argv_free(job->personality); } for (n = 0; n < job->apps->size; n++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(job->apps, n))) { continue; } PMIX_RELEASE(app); } PMIX_RELEASE(job->apps); /* release any pointers in the attributes */ evtimer = NULL; if (prte_get_attribute(&job->attributes, PRTE_JOB_TIMEOUT_EVENT, (void **) &evtimer, PMIX_POINTER)) { prte_event_evtimer_del(evtimer->ev); prte_remove_attribute(&job->attributes, PRTE_JOB_TIMEOUT_EVENT); /* the timer is a pointer to prte_timer_t */ PMIX_RELEASE(evtimer); } evtimer = NULL; if (prte_get_attribute(&job->attributes, PRTE_SPAWN_TIMEOUT_EVENT, (void **) &evtimer, PMIX_POINTER)) { prte_event_evtimer_del(evtimer->ev); prte_remove_attribute(&job->attributes, PRTE_SPAWN_TIMEOUT_EVENT); /* the timer is a pointer to prte_timer_t */ PMIX_RELEASE(evtimer); } proc = NULL; if (prte_get_attribute(&job->attributes, PRTE_JOB_ABORTED_PROC, (void **) &proc, PMIX_POINTER)) { prte_remove_attribute(&job->attributes, PRTE_JOB_ABORTED_PROC); /* points to an prte_proc_t */ PMIX_RELEASE(proc); } if (prte_get_attribute(&job->attributes, PRTE_JOB_INFO_CACHE, (void **) &cache, PMIX_POINTER)) { prte_remove_attribute(&job->attributes, PRTE_JOB_INFO_CACHE); PMIX_LIST_RELEASE(cache); } if (NULL != job->map) { PMIX_RELEASE(job->map); job->map = NULL; } for (n = 0; n < job->procs->size; n++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(job->procs, n))) { continue; } pmix_pointer_array_set_item(job->procs, n, NULL); PMIX_RELEASE(proc); } PMIX_RELEASE(job->procs); /* release the attributes */ PMIX_LIST_DESTRUCT(&job->attributes); PMIX_DATA_BUFFER_DESTRUCT(&job->launch_msg); /* Release the child list - we retained the child jobs before adding them to the list*/ PMIX_LIST_DESTRUCT(&job->children); if (NULL != job->session_dir) { prte_job_session_dir_finalize(job); if (NULL != job->session_dir) { free(job->session_dir); job->session_dir = NULL; } } if (NULL != prte_job_data && 0 <= job->index) { /* remove the job from the global array */ pmix_pointer_array_set_item(prte_job_data, job->index, NULL); } if (NULL != job->traces) { PMIx_Argv_free(job->traces); } PMIX_DESTRUCT(&job->cli); } PMIX_CLASS_INSTANCE(prte_job_t, pmix_list_item_t, prte_job_construct, prte_job_destruct); static void prte_node_construct(prte_node_t *node) { node->index = -1; node->name = NULL; node->rawname = NULL; node->aliases = NULL; node->daemon = NULL; node->available = NULL; node->jobcache = hwloc_bitmap_alloc(); node->num_procs = 0; node->procs = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(node->procs, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); node->next_node_rank = 0; node->state = PRTE_NODE_STATE_UNKNOWN; node->slots = 0; node->slots_available = 0; node->slots_inuse = 0; node->slots_max = 0; node->topology = NULL; node->topodiff = NULL; node->flags = 0; PMIX_CONSTRUCT(&node->attributes, pmix_list_t); } static void prte_node_destruct(prte_node_t *node) { int i; prte_proc_t *proc; if (NULL != node->name) { free(node->name); node->name = NULL; } if (NULL != node->rawname) { free(node->rawname); node->rawname = NULL; } if (NULL != node->aliases) { PMIx_Argv_free(node->aliases); node->aliases = NULL; } if (NULL != node->daemon) { node->daemon->node = NULL; PMIX_RELEASE(node->daemon); node->daemon = NULL; } if (NULL != node->available) { hwloc_bitmap_free(node->available); } if (NULL != node->jobcache) { hwloc_bitmap_free(node->jobcache); } for (i = 0; i < node->procs->size; i++) { if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(node->procs, i))) { pmix_pointer_array_set_item(node->procs, i, NULL); PMIX_RELEASE(proc); } } PMIX_RELEASE(node->procs); /* do NOT destroy the topology */ // release any diffs if (NULL != node->topodiff) { hwloc_topology_diff_destroy(node->topodiff); } /* release the attributes */ PMIX_LIST_DESTRUCT(&node->attributes); } PMIX_CLASS_INSTANCE(prte_node_t, pmix_list_item_t, prte_node_construct, prte_node_destruct); static void prte_proc_construct(prte_proc_t *proc) { proc->name = *PRTE_NAME_INVALID; proc->parent = PMIX_RANK_INVALID; proc->pid = 0; proc->local_rank = PRTE_LOCAL_RANK_INVALID; proc->node_rank = PRTE_NODE_RANK_INVALID; proc->numa_rank = PRTE_LOCAL_RANK_INVALID; proc->app_rank = -1; proc->last_errmgr_state = PRTE_PROC_STATE_UNDEF; proc->state = PRTE_PROC_STATE_UNDEF; proc->app_idx = 0; proc->node = NULL; proc->obj = NULL; proc->cpuset = NULL; proc->exit_code = 0; /* Assume we won't fail unless otherwise notified */ proc->rml_uri = NULL; proc->flags = 0; PMIX_CONSTRUCT(&proc->attributes, pmix_list_t); } static void prte_proc_destruct(prte_proc_t *proc) { if (NULL != proc->node) { PMIX_RELEASE(proc->node); proc->node = NULL; } if (NULL != proc->cpuset) { free(proc->cpuset); proc->cpuset = NULL; } if (NULL != proc->rml_uri) { free(proc->rml_uri); proc->rml_uri = NULL; } PMIX_LIST_DESTRUCT(&proc->attributes); } PMIX_CLASS_INSTANCE(prte_proc_t, pmix_list_item_t, prte_proc_construct, prte_proc_destruct); static void prte_job_map_construct(prte_job_map_t *map) { map->req_mapper = NULL; map->last_mapper = NULL; map->mapping = 0; map->ranking = 0; map->binding = 0; map->rtos_set = false; map->num_new_daemons = 0; map->daemon_vpid_start = PMIX_RANK_INVALID; map->num_nodes = 0; map->nodes = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(map->nodes, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); } static void prte_job_map_destruct(prte_job_map_t *map) { int32_t i; prte_node_t *node; if (NULL != map->req_mapper) { free(map->req_mapper); } if (NULL != map->last_mapper) { free(map->last_mapper); } for (i = 0; i < map->nodes->size; i++) { if (NULL != (node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, i))) { PMIX_RELEASE(node); pmix_pointer_array_set_item(map->nodes, i, NULL); } } PMIX_RELEASE(map->nodes); } PMIX_CLASS_INSTANCE(prte_job_map_t, pmix_object_t, prte_job_map_construct, prte_job_map_destruct); static void prte_attr_cons(prte_attribute_t *p) { p->key = 0; p->local = true; // default to local-only data memset(&p->data, 0, sizeof(p->data)); } static void prte_attr_des(prte_attribute_t *p) { PMIX_VALUE_DESTRUCT(&p->data); } PMIX_CLASS_INSTANCE(prte_attribute_t, pmix_list_item_t, prte_attr_cons, prte_attr_des); static void tcon(prte_topology_t *t) { t->topo = NULL; } static void tdes(prte_topology_t *t) { hwloc_obj_t root; if (NULL != t->topo) { root = hwloc_get_root_obj(t->topo); if (NULL != root->userdata) { PMIX_RELEASE(root->userdata); } hwloc_topology_destroy(t->topo); } } PMIX_CLASS_INSTANCE(prte_topology_t, pmix_object_t, tcon, tdes); static void session_con(prte_session_t *s) { s->index = -1; s->session_id = UINT32_MAX; s->user_refid = NULL; s->alloc_refid = NULL; memset(&s->timeout, 0, sizeof(struct timeval)); s->nodes = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(s->nodes, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); s->jobs = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(s->jobs, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); s->children = PMIX_NEW(pmix_pointer_array_t); pmix_pointer_array_init(s->children, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); } static void session_des(prte_session_t *s) { int n; prte_node_t *nd; prte_job_t *job; prte_session_t *session; if (NULL != s->user_refid) { free(s->user_refid); } if (NULL != s->alloc_refid) { free(s->alloc_refid); } for (n=0; n < s->nodes->size; s++) { nd = (prte_node_t*)pmix_pointer_array_get_item(s->nodes, n); if (NULL != nd) { PMIX_RELEASE(nd); pmix_pointer_array_set_item(s->nodes, n, NULL); } } PMIX_RELEASE(s->nodes); for (n=0; n < s->jobs->size; s++) { job = (prte_job_t*)pmix_pointer_array_get_item(s->jobs, n); if (NULL != job) { PMIX_RELEASE(job); pmix_pointer_array_set_item(s->jobs, n, NULL); } } PMIX_RELEASE(s->jobs); for (n=0; n < s->children->size; s++) { session = (prte_session_t*)pmix_pointer_array_get_item(s->children, n); if (NULL != session) { PMIX_RELEASE(session); pmix_pointer_array_set_item(s->children, n, NULL); } } PMIX_RELEASE(s->children); // remove this from the global array if (0 <= s->index) { pmix_pointer_array_set_item(prte_sessions, s->index, NULL); } } PMIX_CLASS_INSTANCE(prte_session_t, pmix_list_item_t, session_con, session_des); #if PRTE_PICKY_COMPILERS void prte_hide_unused_params(int x, ...) { va_list ap; (void)x; va_start(ap, x); va_end(ap); } #endif prrte-4.1.1~rc2/src/runtime/prte_finalize.c0000664000175000017500000001315215203542132021112 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2020 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/mca/base/pmix_mca_base_alias.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/base/pmix_base.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_progress_threads.h" #include "src/runtime/runtime.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" int prte_finalize(void) { int rc, n, i; prte_job_t *jdata = NULL, *child_jdata = NULL, *next_jdata = NULL; prte_app_context_t *app; prte_proc_t *p; prte_node_t *node; prte_topology_t *topo; PMIX_ACQUIRE_THREAD(&prte_init_lock); if (!prte_initialized) { PMIX_RELEASE_THREAD(&prte_init_lock); return PRTE_ERROR; } prte_initialized = false; PMIX_RELEASE_THREAD(&prte_init_lock); /* protect against multiple calls */ if (pmix_mutex_trylock(&prte_finalize_lock)) { return PRTE_SUCCESS; } #if PRTE_PMIX_STOP_PRGTHRD /* Stop the PMIx server's internal progress thread and wait here * until all active events have been processed */ PMIx_Progress_thread_stop(NULL, 0); #endif /* flag that we are finalizing */ prte_finalizing = true; // stop ALL prte progress threads prte_progress_thread_pause(NULL); // we always must cleanup the session directory tree prte_job_session_dir_finalize(NULL); #ifdef PRTE_PICKY_COMPILERS /* release the cache */ PMIX_RELEASE(prte_cache); /* call the finalize function for this environment */ if (PRTE_SUCCESS != (rc = prte_ess.finalize())) { return rc; } (void) pmix_mca_base_framework_close(&prte_ess_base_framework); // clean up the node array for (n = 0; n < prte_node_pool->size; n++) { node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, n); if (NULL == node) { continue; } pmix_pointer_array_set_item(prte_node_pool, n, NULL); PMIX_RELEASE(node); } PMIX_RELEASE(prte_node_pool); for (n = 0; n < prte_job_data->size; n++) { jdata = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, n); if (NULL == jdata) { continue; } // Remove all children from the list // We do not want to destruct this list here since that occurs in the // prte_job_t destructor - which will happen in the next loop. PMIX_LIST_FOREACH_SAFE(child_jdata, next_jdata, &jdata->children, prte_job_t) { pmix_list_remove_item(&jdata->children, &child_jdata->super); } /* clean up any app contexts as they refcount the jdata object */ for (i=0; i < jdata->apps->size; i++) { app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, i); if (NULL != app) { pmix_pointer_array_set_item(jdata->apps, i, NULL); PMIX_RELEASE(app); } } // clean up any procs for (i=0; i < jdata->procs->size; i++) { p = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, i); if (NULL != p) { pmix_pointer_array_set_item(jdata->procs, i, NULL); PMIX_RELEASE(p); } } pmix_pointer_array_set_item(prte_job_data, n, NULL); PMIX_RELEASE(jdata); } PMIX_RELEASE(prte_job_data); for (n = 0; n < prte_node_topologies->size; n++) { topo = (prte_topology_t *) pmix_pointer_array_get_item(prte_node_topologies, n); if (NULL == topo) { continue; } pmix_pointer_array_set_item(prte_node_topologies, n, NULL); PMIX_RELEASE(topo); } PMIX_RELEASE(prte_node_topologies); /* Close the general debug stream */ pmix_output_close(prte_debug_output); pmix_mca_base_alias_cleanup(); prte_proc_info_finalize(); pmix_output_finalize(); #endif /* now shutdown PMIx - need to do this last as it finalizes * the utilities and class system we depend upon */ PMIx_server_finalize(); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/prte_mca_params.c0000664000175000017500000006642415203542132021426 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #ifdef HAVE_SYS_TIME_H # include #endif #include #include #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/prteinstalldirs/prteinstalldirs.h" #include "src/rml/rml.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_path.h" #include "src/util/pmix_printf.h" #include "src/util/proc_info.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/runtime/runtime.h" static bool passed_thru = false; static int prte_progress_thread_debug_level = -1; static char *prte_tmpdir_base = NULL; static char *prte_local_tmpdir_base = NULL; static char *prte_remote_tmpdir_base = NULL; static char *local_setup_slots = NULL; char *prte_signal_string = NULL; char *prte_stacktrace_output_filename = NULL; char *prte_net_private_ipv4 = NULL; char *prte_if_include = NULL; char *prte_if_exclude = NULL; char *prte_set_max_sys_limits = NULL; int prte_pmix_verbose_output = 0; char *prte_progress_thread_cpus = NULL; bool prte_bind_progress_thread_reqd = false; bool prte_silence_shared_fs = false; int prte_max_thread_in_progress = 1; char *prte_param_files = NULL; char *prte_override_param_file = NULL; bool prte_suppress_override_warning = false; int prte_register_params(void) { int ret; pmix_output_stream_t lds; char *string = NULL; char *fstype = NULL; char *home; char cwd[MAXPATHLEN]; /* only go thru this once - mpirun calls it twice, which causes * any error messages to show up twice */ if (passed_thru) { return PRTE_SUCCESS; } passed_thru = true; /* * This string is going to be used in prte/util/stacktrace.c */ { int j; int signals[] = { #ifdef SIGABRT SIGABRT, #endif #ifdef SIGBUS SIGBUS, #endif #ifdef SIGFPE SIGFPE, #endif #ifdef SIGSEGV SIGSEGV, #endif -1}; for (j = 0; signals[j] != -1; ++j) { if (j == 0) { pmix_asprintf(&string, "%d", signals[j]); } else { char *tmp; pmix_asprintf(&tmp, "%s,%d", string, signals[j]); free(string); string = tmp; } } prte_signal_string = string; ret = pmix_mca_base_var_register("prte", "prte", NULL, "signal", "Comma-delimited list of integer signal numbers to PRTE to attempt to intercept. Upon " "receipt of the intercepted signal, PRTE will display a stack trace and abort. PRTE " "will *not* replace signals if handlers are already installed by the time MPI_INIT is " "invoked. Optionally append \":complain\" to any signal number in the comma-delimited " "list to make PRTE complain if it detects another signal handler (and therefore does " "not insert its own).", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_signal_string); free(string); if (0 > ret) { return ret; } } /* * Where should the stack trace output be directed * This string is going to be used in prte/util/stacktrace.c */ string = strdup("stderr"); prte_stacktrace_output_filename = string; ret = pmix_mca_base_var_register("prte", "prte", NULL, "stacktrace_output", "Specifies where the stack trace output stream goes. " "Accepts one of the following: none (disabled), stderr (default), stdout, file[:filename]. " " " "If 'filename' is not specified, a default filename of 'stacktrace' is used. " "The 'filename' is appended with either '.PID' or '.RANK.PID', if RANK is available. " "The 'filename' can be an absolute path or a relative path to the current working " "directory.", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_stacktrace_output_filename); free(string); if (0 > ret) { return ret; } /* RFC1918 defines - 10.0.0./8 - 172.16.0.0/12 - 192.168.0.0/16 RFC3330 also mentions - 169.254.0.0/16 for DHCP onlink iff there's no DHCP server */ prte_net_private_ipv4 = "10.0.0.0/8;172.16.0.0/12;192.168.0.0/16;169.254.0.0/16"; ret = pmix_mca_base_var_register("prte", "prte", "net", "private_ipv4", "Semicolon-delimited list of CIDR notation entries specifying what networks are considered " "\"private\" (default value based on RFC1918 and RFC3330)", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_net_private_ipv4); if (0 > ret) { return ret; } prte_if_include = NULL; ret = pmix_mca_base_var_register("prte", "prte", NULL, "if_include", "Comma-delimited list of devices and/or CIDR notation of TCP networks to use for PRTE " "bootstrap communication (e.g., \"eth0,192.168.0.0/16\"). Mutually exclusive with " "prte_if_exclude.", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_if_include); (void) pmix_mca_base_var_register_synonym(ret, "prte", "oob", "tcp", "include", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "oob", "tcp", "if_include", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); prte_if_exclude = NULL; ret = pmix_mca_base_var_register("prte", "prte", NULL, "if_exclude", "Comma-delimited list of devices and/or CIDR notation of TCP networks to NOT use for PRTE " "bootstrap communication -- all devices not matching these specifications will be used " "(e.g., \"eth0,192.168.0.0/16\"). If set to a non-default value, it is mutually exclusive " "with prte_if_include.", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_if_exclude); (void) pmix_mca_base_var_register_synonym(ret, "prte", "oob", "tcp", "exclude", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "oob", "tcp", "if_exclude", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); /* if_include and if_exclude need to be mutually exclusive */ if (NULL != prte_if_include && NULL != prte_if_exclude) { /* Return ERR_NOT_AVAILABLE so that a warning message about "open" failing is not printed */ pmix_show_help("help-oob-tcp.txt", "include-exclude", true, prte_if_include, prte_if_exclude); return PRTE_ERR_SILENT; } prte_set_max_sys_limits = NULL; ret = pmix_mca_base_var_register("prte", "prte", NULL, "set_max_sys_limits", "Set the specified system-imposed limits to the specified value, including \"unlimited\"." "Supported params: core, filesize, maxmem, openfiles, stacksize, maxchildren", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_set_max_sys_limits); if (0 > ret) { return ret; } /* get a clean output channel too - need to do this here because * we use it below, and prun and some other tools call this * function prior to calling prte_init */ PMIX_CONSTRUCT(&lds, pmix_output_stream_t); lds.lds_want_stdout = true; prte_clean_output = pmix_output_open(&lds); PMIX_DESTRUCT(&lds); /* check directive for warning about shared fs on tmpdir */ (void) pmix_mca_base_var_register("prte", "prte", NULL, "silence_shared_fs", "Silence the shared file system warning", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_silence_shared_fs); /* LOOK FOR A TMP DIRECTORY BASE */ /* Several options are provided to cover a range of possibilities: * * (a) all processes need to use a specified location as the base * for tmp directories * (b) daemons on remote nodes need to use a specified location, but * one different from that used by mpirun * (c) mpirun needs to use a specified location, but one different * from that used on remote nodes */ prte_tmpdir_base = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "tmpdir_base", "Base of the session directory tree to be used by all processes", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_tmpdir_base); prte_local_tmpdir_base = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "local_tmpdir_base", "Base of the session directory tree to be used by prun/mpirun", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_local_tmpdir_base); prte_remote_tmpdir_base = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "remote_tmpdir_base", "Base of the session directory tree on remote nodes, if " "required to be different from head node", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_remote_tmpdir_base); /* if a global tmpdir was specified, then we do not allow specification * of the local or remote values to avoid confusion */ if (NULL != prte_tmpdir_base && (NULL != prte_local_tmpdir_base || NULL != prte_remote_tmpdir_base)) { pmix_output(prte_clean_output, "------------------------------------------------------------------\n" "The MCA param prte_tmpdir_base was specified, which sets the base\n" "of the temporary directory tree for all procs. However, values for\n" "the local and/or remote tmpdir base were also given. This can lead\n" "to confusion and is therefore not allowed. Please specify either a\n" "global tmpdir base OR a local/remote tmpdir base value\n" "------------------------------------------------------------------"); exit(1); } if (NULL != prte_tmpdir_base) { if (NULL != prte_process_info.tmpdir_base) { free(prte_process_info.tmpdir_base); } prte_process_info.tmpdir_base = strdup(prte_tmpdir_base); } else if (PRTE_PROC_IS_MASTER && NULL != prte_local_tmpdir_base) { /* prun will pickup the value for its own use */ if (NULL != prte_process_info.tmpdir_base) { free(prte_process_info.tmpdir_base); } prte_process_info.tmpdir_base = strdup(prte_local_tmpdir_base); } else if (PRTE_PROC_IS_DAEMON && NULL != prte_remote_tmpdir_base) { /* prun will pickup the value and forward it along, but must not * use it in its own work. So only a daemon needs to get it, and the * daemon will pass it down to its application procs. Note that prun * will pass -its- value to any procs local to it */ if (NULL != prte_process_info.tmpdir_base) { free(prte_process_info.tmpdir_base); } prte_process_info.tmpdir_base = strdup(prte_remote_tmpdir_base); } else { if (NULL != prte_process_info.tmpdir_base) { free(prte_process_info.tmpdir_base); } prte_process_info.tmpdir_base = strdup(pmix_tmp_directory()); } // check to see if this is on a shared file system // as we know this will impact launch as well as // application execution performance prte_process_info.shared_fs = pmix_path_nfs(prte_process_info.tmpdir_base, &fstype); if (prte_process_info.shared_fs && !prte_silence_shared_fs) { // this is a shared file system - warn the user pmix_show_help("help-prte-runtime.txt", "prte:session:dir:shared", true, prte_process_info.tmpdir_base, fstype, prte_tool_basename); } if (NULL != fstype) { free(fstype); } prte_prohibited_session_dirs = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "no_session_dirs", "Prohibited locations for session directories (multiple " "locations separated by ',', default=NULL)", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_prohibited_session_dirs); prte_fwd_environment = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "fwd_environment", "Forward the entire local environment", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_fwd_environment); prte_execute_quiet = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "execute_quiet", "Do not output error and help messages", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_execute_quiet); prte_report_silent_errors = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "report_silent_errors", "Report all errors, including silent ones", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_report_silent_errors); prte_progress_thread_debug_level = -1; (void) pmix_mca_base_var_register("prte", "prte", NULL, "progress_thread_debug", "Debug level for PRTE progress threads", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_progress_thread_debug_level); if (0 <= prte_progress_thread_debug_level) { prte_progress_thread_debug = pmix_output_open(NULL); pmix_output_set_verbosity(prte_progress_thread_debug, prte_progress_thread_debug_level); } prted_debug_failure = PMIX_RANK_INVALID; (void) pmix_mca_base_var_register("prte", "prte", NULL, "daemon_fail", "Have the specified prted fail after init for debugging purposes", PMIX_MCA_BASE_VAR_TYPE_INT, &prted_debug_failure); prted_debug_failure_delay = 0; (void) pmix_mca_base_var_register("prte", "prte", NULL, "daemon_fail_delay", "Have the specified prted fail after specified number of seconds [default: 0 => no delay]", PMIX_MCA_BASE_VAR_TYPE_INT, &prted_debug_failure_delay); /* default hostfile */ prte_default_hostfile = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "default_hostfile", "Name of the default hostfile (relative or absolute path, " "\"none\" to ignore environmental or default MCA param setting)", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_default_hostfile); if (NULL == prte_default_hostfile) { /* nothing was given, so define the default */ pmix_asprintf(&prte_default_hostfile, "%s/prte-default-hostfile", prte_install_dirs.sysconfdir); /* flag that nothing was given */ prte_default_hostfile_given = false; } else if (0 == strcmp(prte_default_hostfile, "none")) { free(prte_default_hostfile); prte_default_hostfile = NULL; /* flag that it was given */ prte_default_hostfile_given = true; } else { /* flag that it was given */ prte_default_hostfile_given = true; } /* default dash-host */ prte_default_dash_host = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "default_dash_host", "Default -host setting (specify \"none\" to ignore " "environmental or default MCA param setting)", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_default_dash_host); if (NULL != prte_default_dash_host && 0 == strcmp(prte_default_dash_host, "none")) { free(prte_default_dash_host); prte_default_dash_host = NULL; } prte_show_resolved_nodenames = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "show_resolved_nodenames", "Display any node names that are resolved to a different name [default: false]", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_show_resolved_nodenames); prte_do_not_resolve = true; (void) pmix_mca_base_var_register("prte", "prte", NULL, "do_not_resolve", "Do not attempt to resolve hostnames " "[defaults to true]", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_do_not_resolve); /* allow specification of the launch agent */ prte_launch_agent = "prted"; (void) pmix_mca_base_var_register("prte", "prte", NULL, "launch_agent", "Executable for DVM daemons on remote nodes [default: prted]", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_launch_agent); /* whether or not to require RM allocation */ prte_allocation_required = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "allocation_required", "Whether or not an allocation by a resource manager is required [default: no]", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_allocation_required); prte_allowed_exit_without_sync = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "allowed_exit_without_sync", "Set default process exiting without calling finalize policy to not trigger job termination", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_allowed_exit_without_sync); prte_report_child_jobs_separately = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "report_child_jobs_separately", "Set default to return the exit status of the primary job only", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_report_child_jobs_separately); prte_stat_history_size = 1; (void) pmix_mca_base_var_register("prte", "prte", NULL, "stat_history_size", "Number of stat samples to keep", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_stat_history_size); prte_max_vm_size = -1; (void) pmix_mca_base_var_register("prte", "prte", NULL, "max_vm_size", "Maximum size of virtual machine - used to subdivide allocation", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_max_vm_size); local_setup_slots = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "set_default_slots", "Set the number of slots on nodes that lack such info to the" " number of specified objects [a number, \"cores\" (default)," " \"packages\", or \"hwthreads\" (default if hwthreads_as_cpus" " is set), or a fixed number to be applied to all nodes", PMIX_MCA_BASE_VAR_TYPE_STRING, &local_setup_slots); if (NULL == local_setup_slots) { prte_set_slots = strdup("core"); } else { prte_set_slots = strdup(local_setup_slots); } prte_set_slots_override = false; (void) pmix_mca_base_var_register("prte", "prte", NULL, "set_default_slots_override", "Set the number of slots on nodes to the number of " "objects specified by prte_set_default_slots regardless " "whather we are in a managed allocation or specifications " "were given in a hostfile", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_set_slots_override); /* allow specification of the cores to be used by daemons */ prte_daemon_cores = NULL; (void) pmix_mca_base_var_register("prte", "prte", NULL, "daemon_cores", "Restrict the PRTE daemons (including mpirun) to operate on " "the specified cores (comma-separated list of ranges)", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_daemon_cores); /* Amount of time to wait for a stack trace to return from the daemons */ prte_stack_trace_wait_timeout = 30; (void) pmix_mca_base_var_register("prte", "prte", NULL, "timeout_for_stack_trace", "Seconds to wait for stack traces to return before terminating " "the job (<= 0 wait forever)", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_stack_trace_wait_timeout); /* register the URI of the UNIVERSAL data server */ prte_data_server_uri = NULL; (void) pmix_mca_base_var_register("prte", "pmix", NULL, "server_uri", "URI of a session-level keyval server for publish/lookup operations", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_data_server_uri); pmix_mca_base_var_register("prte", "prte", NULL, "pmix_verbose", "Verbosity for PRRTE-level PMIx code", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_pmix_verbose_output); (void) pmix_mca_base_var_register("prte", "prte", NULL, "progress_thread_cpus", "Comma-delimited list of ranges of CPUs to which " "the internal PRRTE progress thread is to be bound", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_progress_thread_cpus); (void) pmix_mca_base_var_register("prte", "prte", NULL, "bind_progress_thread_reqd", "Whether binding of internal PRRTE progress thread is required", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_bind_progress_thread_reqd); (void) pmix_mca_base_var_register("prte", "prte", NULL, "uniform_nodes", "Allocation contains homogeneous nodes", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_homo_nodes); home = (char *) pmix_home_directory(geteuid()); if (NULL == getcwd(cwd, MAXPATHLEN)) { return PRTE_ERROR; } #if PRTE_WANT_HOME_CONFIG_FILES pmix_asprintf(&prte_param_files, "%s" PMIX_PATH_SEP ".prte" PMIX_PATH_SEP "mca-params.conf%c%s" PMIX_PATH_SEP "prte-mca-params.conf", home, ',', prte_install_dirs.sysconfdir); #else pmix_sprintf(&prte_param_files, "%s" PMIX_PATH_SEP "prte-mca-params.conf", prte_install_dirs.sysconfdir); #endif (void) pmix_mca_base_var_register("prte", "prte", NULL, "param_files", "Path for MCA configuration files containing " "variable values", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_param_files); (void) pmix_mca_base_var_register("prte", "prte", NULL, "override_param_file", "Variables set in this file will override any value " "set in the environment or another configuration file", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_override_param_file); (void) pmix_mca_base_var_register("prte", "prte", NULL, "suppress_override_warning", "Suppress warnings when attempting to set an " "overridden value (default: false)", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_suppress_override_warning); /* pickup the RML params */ prte_rml_register(); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/runtime/prte_quit.h0000664000175000017500000000126415203542132020301 0ustar alastairalastair/* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef PRTE_QUIT_H #define PRTE_QUIT_H #include "prte_config.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS PRTE_EXPORT void prte_quit(int fd, short args, void *cbdata); PRTE_EXPORT char *prte_dump_aborted_procs(prte_job_t *jdata); END_C_DECLS #endif /* PRTE_CR_H */ prrte-4.1.1~rc2/src/runtime/prte_progress_threads.c0000664000175000017500000003047415203542132022675 0ustar alastairalastair/* * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_UNISTD_H # include #endif #include #include #ifdef HAVE_PTHREAD_NP_H # include #endif #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_argv.h" #include "src/util/error.h" #include "src/util/pmix_fd.h" #include "src/runtime/prte_progress_threads.h" /* create a tracking object for progress threads */ typedef struct { pmix_list_item_t super; int refcount; char *name; prte_event_base_t *ev_base; /* This will be set to false when it is time for the progress thread to exit */ volatile bool ev_active; /* This event will always be set on the ev_base (so that the ev_base is not empty!) */ prte_event_t block; bool engine_constructed; pmix_thread_t engine; #if PRTE_HAVE_LIBEV ev_async async; pthread_mutex_t mutex; pthread_cond_t cond; pmix_list_t list; #endif } prte_progress_tracker_t; static void tracker_constructor(prte_progress_tracker_t *p) { p->refcount = 1; // start at one since someone created it p->name = NULL; p->ev_base = NULL; p->ev_active = false; p->engine_constructed = false; #if PRTE_HAVE_LIBEV pthread_mutex_init(&p->mutex, NULL); PMIX_CONSTRUCT(&p->list, pmix_list_t); #endif } static void tracker_destructor(prte_progress_tracker_t *p) { prte_event_del(&p->block); if (NULL != p->name) { free(p->name); } if (NULL != p->ev_base) { prte_event_base_free(p->ev_base); } if (p->engine_constructed) { PMIX_DESTRUCT(&p->engine); } #if PRTE_HAVE_LIBEV pthread_mutex_destroy(&p->mutex); PMIX_LIST_DESTRUCT(&p->list); #endif } static PMIX_CLASS_INSTANCE(prte_progress_tracker_t, pmix_list_item_t, tracker_constructor, tracker_destructor); #if PRTE_HAVE_LIBEV typedef enum { PRTE_EVENT_ACTIVE, PRTE_EVENT_ADD, PRTE_EVENT_DEL } prte_event_type_t; typedef struct { pmix_list_item_t super; struct event *ev; struct timeval *tv; int res; short ncalls; prte_event_type_t type; } prte_event_caddy_t; static PMIX_CLASS_INSTANCE(prte_event_caddy_t, pmix_list_item_t, NULL, NULL); static prte_progress_tracker_t *prte_progress_tracker_get_by_base(struct event_base *); static void prte_libev_ev_async_cb(EV_P_ ev_async *w, int revents) { prte_progress_tracker_t *trk = prte_progress_tracker_get_by_base((struct event_base *) EV_A); assert(NULL != trk); pthread_mutex_lock(&trk->mutex); prte_event_caddy_t *cd, *next; PMIX_LIST_FOREACH_SAFE(cd, next, &trk->list, prte_event_caddy_t) { switch (cd->type) { case PRTE_EVENT_ADD: (void) event_add(cd->ev, cd->tv); break; case PRTE_EVENT_DEL: (void) event_del(cd->ev); break; case PRTE_EVENT_ACTIVE: (void) event_active(cd->ev, cd->res, cd->ncalls); break; } pmix_list_remove_item(&trk->list, &cd->super); PMIX_RELEASE(cd); } pthread_mutex_unlock(&trk->mutex); } int prte_event_add(struct event *ev, struct timeval *tv) { int res; prte_progress_tracker_t *trk = prte_progress_tracker_get_by_base(ev->ev_base); if ((NULL != trk) && !pthread_equal(pthread_self(), trk->engine.t_handle)) { prte_event_caddy_t *cd = PMIX_NEW(prte_event_caddy_t); cd->type = PRTE_EVENT_ADD; cd->ev = ev; cd->tv = tv; pthread_mutex_lock(&trk->mutex); pmix_list_append(&trk->list, &cd->super); ev_async_send((struct ev_loop *) trk->ev_base, &trk->async); pthread_mutex_unlock(&trk->mutex); res = PRTE_SUCCESS; } else { res = event_add(ev, tv); } return res; } int prte_event_del(struct event *ev) { int res; prte_progress_tracker_t *trk = prte_progress_tracker_get_by_base(ev->ev_base); if ((NULL != trk) && !pthread_equal(pthread_self(), trk->engine.t_handle)) { prte_event_caddy_t *cd = PMIX_NEW(prte_event_caddy_t); cd->type = PRTE_EVENT_DEL; cd->ev = ev; pthread_mutex_lock(&trk->mutex); pmix_list_append(&trk->list, &cd->super); ev_async_send((struct ev_loop *) trk->ev_base, &trk->async); pthread_mutex_unlock(&trk->mutex); res = PRTE_SUCCESS; } else { res = event_del(ev); } return res; } void prte_event_active(struct event *ev, int res, short ncalls) { prte_progress_tracker_t *trk = prte_progress_tracker_get_by_base(ev->ev_base); if ((NULL != trk) && !pthread_equal(pthread_self(), trk->engine.t_handle)) { prte_event_caddy_t *cd = PMIX_NEW(prte_event_caddy_t); cd->type = PRTE_EVENT_ACTIVE; cd->ev = ev; cd->res = res; cd->ncalls = ncalls; pthread_mutex_lock(&trk->mutex); pmix_list_append(&trk->list, &cd->super); ev_async_send((struct ev_loop *) trk->ev_base, &trk->async); pthread_mutex_unlock(&trk->mutex); } else { event_active(ev, res, ncalls); } } void prte_event_base_loopexit(prte_event_base_t *ev_base) { prte_progress_tracker_t *trk = prte_progress_tracker_get_by_base(ev_base); assert(NULL != trk); ev_async_send((struct ev_loop *) trk->ev_base, &trk->async); } #endif static bool inited = false; static pmix_list_t tracking; static struct timeval long_timeout = {.tv_sec = 3600, .tv_usec = 0}; static const char *shared_thread_name = "PRTE-wide async progress thread"; /* * If this event is fired, just restart it so that this event base * continues to have something to block on. */ static void dummy_timeout_cb(int fd, short args, void *cbdata) { prte_progress_tracker_t *trk = (prte_progress_tracker_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); prte_event_add(&trk->block, &long_timeout); } /* * Main for the progress thread */ static void *progress_engine(pmix_object_t *obj) { pmix_thread_t *t = (pmix_thread_t *) obj; prte_progress_tracker_t *trk = (prte_progress_tracker_t *) t->t_arg; while (trk->ev_active) { prte_event_loop(trk->ev_base, PRTE_EVLOOP_ONCE); } return PMIX_THREAD_CANCELLED; } static void stop_progress_engine(prte_progress_tracker_t *trk) { assert(trk->ev_active); trk->ev_active = false; /* break the event loop - this will cause the loop to exit upon completion of any current event */ prte_event_base_loopexit(trk->ev_base); pmix_thread_join(&trk->engine, NULL); } static int start_progress_engine(prte_progress_tracker_t *trk) { #ifdef HAVE_PTHREAD_SETAFFINITY_NP cpu_set_t cpuset; char **ranges, *dash; int k, n, start, end; #endif assert(!trk->ev_active); trk->ev_active = true; /* fork off a thread to progress it */ trk->engine.t_run = progress_engine; trk->engine.t_arg = trk; int rc = pmix_thread_start(&trk->engine); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } #ifdef HAVE_PTHREAD_SETAFFINITY_NP if (NULL != prte_progress_thread_cpus) { CPU_ZERO(&cpuset); // comma-delimited list of cpu ranges ranges = PMIx_Argv_split(prte_progress_thread_cpus, ','); for (n=0; NULL != ranges[n]; n++) { // look for '-' start = strtoul(ranges[n], &dash, 10); if (NULL == dash) { CPU_SET(start, &cpuset); } else { ++dash; // skip over the '-' end = strtoul(dash, NULL, 10); for (k=start; k < end; k++) { CPU_SET(k, &cpuset); } } } rc = pthread_setaffinity_np(trk->engine.t_handle, sizeof(cpu_set_t), &cpuset); if (0 != rc && prte_bind_progress_thread_reqd) { pmix_output(0, "Failed to bind progress thread %s", (NULL == trk->name) ? "NULL" : trk->name); rc = PRTE_ERR_NOT_SUPPORTED; } else { rc = PRTE_SUCCESS; } } #endif return rc; } prte_event_base_t *prte_progress_thread_init(const char *name) { prte_progress_tracker_t *trk; int rc; if (!inited) { PMIX_CONSTRUCT(&tracking, pmix_list_t); inited = true; } if (NULL == name) { name = shared_thread_name; } /* check if we already have this thread */ PMIX_LIST_FOREACH(trk, &tracking, prte_progress_tracker_t) { if (0 == strcmp(name, trk->name)) { /* we do, so up the refcount on it */ ++trk->refcount; /* return the existing base */ return trk->ev_base; } } trk = PMIX_NEW(prte_progress_tracker_t); if (NULL == trk) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return NULL; } trk->name = strdup(name); if (NULL == trk->name) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); PMIX_RELEASE(trk); return NULL; } if (NULL == (trk->ev_base = prte_event_base_create())) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); PMIX_RELEASE(trk); return NULL; } /* add an event to the new event base (if there are no events, prte_event_loop() will return immediately) */ prte_event_set(trk->ev_base, &trk->block, -1, PRTE_EV_PERSIST, dummy_timeout_cb, trk); prte_event_add(&trk->block, &long_timeout); #if PRTE_HAVE_LIBEV ev_async_init(&trk->async, prte_libev_ev_async_cb); ev_async_start((struct ev_loop *) trk->ev_base, &trk->async); #endif /* construct the thread object */ PMIX_CONSTRUCT(&trk->engine, pmix_thread_t); trk->engine_constructed = true; if (PRTE_SUCCESS != (rc = start_progress_engine(trk))) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(trk); return NULL; } pmix_list_append(&tracking, &trk->super); return trk->ev_base; } int prte_progress_thread_finalize(const char *name) { prte_progress_tracker_t *trk; if (!inited) { /* nothing we can do */ return PRTE_ERR_NOT_FOUND; } if (NULL == name) { name = shared_thread_name; } /* find the specified engine */ PMIX_LIST_FOREACH(trk, &tracking, prte_progress_tracker_t) { if (0 == strcmp(name, trk->name)) { /* If the progress thread is active, stop it */ if (trk->ev_active) { stop_progress_engine(trk); } pmix_list_remove_item(&tracking, &trk->super); PMIX_RELEASE(trk); return PRTE_SUCCESS; } } return PRTE_ERR_NOT_FOUND; } /* * Stop the progress thread, but don't delete the tracker (or event base) */ int prte_progress_thread_pause(const char *name) { prte_progress_tracker_t *trk; if (!inited) { /* nothing we can do */ return PRTE_ERR_NOT_FOUND; } /* find the specified engine */ PMIX_LIST_FOREACH(trk, &tracking, prte_progress_tracker_t) { if (NULL == name || 0 == strcmp(name, trk->name)) { if (trk->ev_active) { stop_progress_engine(trk); } if (NULL != name) { break; } } } return PRTE_SUCCESS; } #if PRTE_HAVE_LIBEV static prte_progress_tracker_t *prte_progress_tracker_get_by_base(prte_event_base_t *base) { prte_progress_tracker_t *trk; if (inited) { PMIX_LIST_FOREACH(trk, &tracking, prte_progress_tracker_t) { if (trk->ev_base == base) { return trk; } } } return NULL; } #endif int prte_progress_thread_resume(const char *name) { prte_progress_tracker_t *trk; if (!inited) { /* nothing we can do */ return PRTE_ERR_NOT_FOUND; } if (NULL == name) { name = shared_thread_name; } /* find the specified engine */ PMIX_LIST_FOREACH(trk, &tracking, prte_progress_tracker_t) { if (0 == strcmp(name, trk->name)) { if (trk->ev_active) { return PRTE_ERR_RESOURCE_BUSY; } return start_progress_engine(trk); } } return PRTE_ERR_NOT_FOUND; } prrte-4.1.1~rc2/src/runtime/data_server/0000775000175000017500000000000015203542132020410 5ustar alastairalastairprrte-4.1.1~rc2/src/runtime/data_server/ds_purge.c0000664000175000017500000000671015203542132022370 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2016 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2025 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/data_server/ds.h" void prte_ds_purge(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer) { int32_t count; prte_data_object_t *data; int k; pmix_status_t rc; pmix_proc_t requestor; /* unpack the proc whose data is to be purged - session * data is purged by providing a requestor whose rank * is wildcard */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &requestor, &count, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; } pmix_output_verbose(1, prte_data_store.output, "%s data server: purge data from %s:%d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), requestor.nspace, requestor.rank); /* cycle across the stored data, looking for a match */ for (k = 0; k < prte_data_store.store.size; k++) { data = (prte_data_object_t *) pmix_pointer_array_get_item(&prte_data_store.store, k); if (NULL == data) { continue; } /* check if data posted by the specified process */ if (!PMIX_CHECK_PROCID(&requestor, &data->owner)) { continue; } /* remove the object */ pmix_pointer_array_set_item(&prte_data_store.store, data->index, NULL); PMIX_RELEASE(data); } done: // send back an answer rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } PRTE_RML_SEND(rc, sender->rank, answer, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); } } prrte-4.1.1~rc2/src/runtime/data_server/ds_lookup.c0000664000175000017500000002643715203542132022567 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2016 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2025 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/data_server/ds.h" pmix_status_t prte_ds_lookup(pmix_proc_t *sender, int room_number, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer) { int32_t count; int i, k; size_t nanswers; pmix_status_t rc; pmix_proc_t requestor; size_t n, ninfo; char **keys = NULL, **cache = NULL; char *str; pmix_info_t *info; pmix_data_buffer_t pbkt; uint32_t uid = UINT32_MAX; bool wait = false; pmix_data_range_t range=PMIX_RANGE_UNDEF; prte_data_object_t *data; prte_ds_info_t *rinfo; prte_info_item_t *ds1, *ds2; pmix_list_t answers; bool found; prte_data_req_t *req, rq; pmix_byte_object_t pbo; /* unpack the requestor */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &requestor, &count, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* unpack the number of keys */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &ninfo, &count, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } if (0 == ninfo) { /* they forgot to send us the keys?? */ PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return PMIX_ERR_BAD_PARAM; } /* unpack the keys */ for (n = 0; n < ninfo; n++) { count = 1; rc = PMIx_Data_unpack(NULL, buffer, &str, &count, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIx_Argv_free(keys); return rc; } PMIx_Argv_append_nosize(&keys, str); free(str); } /* unpack the number of directives, if any */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &ninfo, &count, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIx_Argv_free(keys); return rc; } if (0 < ninfo) { PMIX_INFO_CREATE(info, ninfo); count = ninfo; rc = PMIx_Data_unpack(NULL, buffer, info, &count, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); PMIx_Argv_free(keys); return rc; } /* scan the directives for things we care about */ for (n = 0; n < ninfo; n++) { if (PMIx_Check_key(info[n].key, PMIX_USERID)) { uid = info[n].value.data.uint32; } else if (PMIx_Check_key(info[n].key, PMIX_WAIT)) { /* flag that we wait until the data is present */ wait = true; } else if (PMIx_Check_key(info[n].key, PMIX_RANGE)) { range = info[n].value.data.range; } } /* ignore anything else for now */ PMIX_INFO_FREE(info, ninfo); } /* cycle across the provided keys */ PMIX_DATA_BUFFER_CONSTRUCT(&pbkt); PMIX_CONSTRUCT(&answers, pmix_list_t); PMIX_CONSTRUCT(&rq, prte_data_req_t); memcpy(&rq.requestor, &requestor, sizeof(pmix_proc_t)); memcpy(&rq.proxy, sender, sizeof(pmix_proc_t)); for (i = 0; NULL != keys[i]; i++) { pmix_output_verbose(10, prte_data_store.output, "%s data server: looking for %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), keys[i]); found = false; /* cycle across the stored data, looking for a match */ for (k = 0; k < prte_data_store.store.size && !found; k++) { data = (prte_data_object_t *) pmix_pointer_array_get_item(&prte_data_store.store, k); if (NULL == data) { continue; } /* for security reasons, can only access data posted by the same user id */ if (uid != data->uid) { pmix_output_verbose(10, prte_data_store.output, "%s\tMISMATCH UID %u %u", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (unsigned) uid, (unsigned) data->uid); continue; } /* check the range */ if (PMIX_SUCCESS != prte_data_server_check_range(&rq, data)) { continue; } /* see if we have this key */ PMIX_LIST_FOREACH_SAFE(ds1, ds2, &data->info, prte_info_item_t) { pmix_output_verbose(10, prte_data_store.output, "%s COMPARING %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), keys[i], ds1->info.key); if (PMIx_Check_key(ds1->info.key, keys[i])) { rinfo = PMIX_NEW(prte_ds_info_t); memcpy(&rinfo->source, &data->owner, sizeof(pmix_proc_t)); PMIX_INFO_XFER(&rinfo->info, &ds1->info); // check the persistence pmix_output_verbose(1, prte_data_store.output, "%s data server: adding %s to data from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), ds1->info.key, PRTE_NAME_PRINT(&data->owner)); if (PMIX_PERSIST_FIRST_READ == data->persistence) { pmix_list_remove_item(&data->info, &ds1->super); PMIX_RELEASE(ds1); } pmix_list_append(&answers, &rinfo->super); // can only find it once - keys are required to be globally unique // within a given range, and we checked the range above found = true; break; } } } // loop over stored data if (!found) { // cache the key PMIx_Argv_append_nosize(&cache, keys[i]); } } // loop over keys nanswers = pmix_list_get_size(&answers); rc = PMIX_ERR_NOT_FOUND; if (0 < nanswers) { /* pack the number of data items found */ rc = PMIx_Data_pack(NULL, &pbkt, &nanswers, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_LIST_DESTRUCT(&answers); PMIx_Argv_free(keys); PMIx_Argv_free(cache); return rc; } /* loop thru and pack the individual responses - this is somewhat less * efficient than packing an info array, but avoids another malloc * operation just to assemble all the return values into a contiguous * array */ PMIX_LIST_FOREACH(rinfo, &answers, prte_ds_info_t) { /* pack the data owner */ rc = PMIx_Data_pack(NULL, &pbkt, &rinfo->source, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_LIST_DESTRUCT(&answers); PMIx_Argv_free(keys); PMIx_Argv_free(cache); return rc; } rc = PMIx_Data_pack(NULL, &pbkt, &rinfo->info, 1, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_LIST_DESTRUCT(&answers); PMIx_Argv_free(keys); PMIx_Argv_free(cache); return rc; } } } PMIX_LIST_DESTRUCT(&answers); i = PMIx_Argv_count(cache); if (0 < i) { if (wait) { pmix_output_verbose(1, prte_data_store.output, "%s data server:lookup: at least some data not found %d vs %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) nanswers, (int) PMIx_Argv_count(keys)); req = PMIX_NEW(prte_data_req_t); req->room_number = room_number; req->proxy = *sender; memcpy(&req->requestor, &requestor, sizeof(pmix_proc_t)); req->uid = uid; req->range = range; req->keys = cache; cache = NULL; pmix_list_append(&prte_data_store.pending, &req->super); PMIx_Argv_free(keys); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return PMIX_SUCCESS; // do not return an answer } else { PMIx_Argv_free(cache); if (0 == nanswers) { /* nothing was found - indicate that situation */ rc = PMIX_ERR_NOT_FOUND; PMIx_Argv_free(keys); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return rc; } else { rc = PMIX_ERR_PARTIAL_SUCCESS; } } } PMIx_Argv_free(keys); pmix_output_verbose(1, prte_data_store.output, "%s data server:lookup: data found - status %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIx_Error_string(rc)); if (PMIX_SUCCESS == rc) { /* pack the status */ rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return rc; } /* unload the packed values */ rc = PMIx_Data_unload(&pbkt, &pbo); /* pack it into our reply */ rc = PMIx_Data_pack(NULL, answer, &pbo, 1, PMIX_BYTE_OBJECT); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } PRTE_RML_SEND(rc, sender->rank, answer, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); } } return rc; } prrte-4.1.1~rc2/src/runtime/data_server/ds_main.c0000664000175000017500000002457215203542132022200 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2016 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2025 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/data_server/ds.h" // globals prte_data_store_t prte_data_store = { .store = PMIX_POINTER_ARRAY_STATIC_INIT, .pending = PMIX_LIST_STATIC_INIT, .output = -1, .verbosity = 0 }; /* locals */ static bool initialized = false; int prte_data_server_init(void) { pmix_status_t rc; if (initialized) { return PRTE_SUCCESS; } initialized = true; /* register a verbosity */ prte_data_store.verbosity = -1; (void) pmix_mca_base_var_register("prte", "prte", "data", "server_verbose", "Debug verbosity for PRTE data server", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_data_store.verbosity); if (0 <= prte_data_store.verbosity) { prte_data_store.output = pmix_output_open(NULL); pmix_output_set_verbosity(prte_data_store.output, prte_data_store.verbosity); } PMIX_CONSTRUCT(&prte_data_store.store, pmix_pointer_array_t); if (PMIX_SUCCESS != (rc = pmix_pointer_array_init(&prte_data_store.store, 1, INT_MAX, 1))) { PMIX_ERROR_LOG(rc); return rc; } PMIX_CONSTRUCT(&prte_data_store.pending, pmix_list_t); PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_DATA_SERVER, PRTE_RML_PERSISTENT, prte_data_server, NULL); return PRTE_SUCCESS; } void prte_data_server_finalize(void) { int32_t i; prte_data_object_t *data; if (!initialized) { return; } initialized = false; for (i = 0; i < prte_data_store.store.size; i++) { data = (prte_data_object_t *) pmix_pointer_array_get_item(&prte_data_store.store, i); if (NULL != data) { PMIX_RELEASE(data); } } PMIX_DESTRUCT(&prte_data_store.store); PMIX_LIST_DESTRUCT(&prte_data_store.pending); } void prte_data_server(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { uint8_t command; int32_t count; pmix_data_buffer_t *answer; pmix_status_t rc; int room_number; PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata); pmix_output_verbose(1, prte_data_store.output, "%s data server got message from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)); /* unpack the room number of the caller's request */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &room_number, &count, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } /* unpack the command */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &command, &count, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } PMIX_DATA_BUFFER_CREATE(answer); /* pack the room number as this must lead any response */ rc = PMIx_Data_pack(NULL, answer, &room_number, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return; } /* and the command */ rc = PMIx_Data_pack(NULL, answer, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return; } switch (command) { case PRTE_PMIX_PUBLISH_CMD: rc = prte_ds_publish(sender, buffer, answer); break; case PRTE_PMIX_LOOKUP_CMD: pmix_output_verbose(1, prte_data_store.output, "%s data server: lookup data from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)); rc = prte_ds_lookup(sender, room_number, buffer, answer); break; case PRTE_PMIX_UNPUBLISH_CMD: rc = prte_ds_unpublish(sender, buffer, answer); break; case PRTE_PMIX_PURGE_PROC_CMD: prte_ds_purge(sender, buffer, answer); return; default: PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); rc = PRTE_ERR_BAD_PARAM; break; } if (PMIX_SUCCESS != rc) { pmix_output_verbose(1, prte_data_store.output, "%s data server: sending error %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_ERROR_NAME(rc)); /* pack the error code */ rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } PRTE_RML_SEND(rc, sender->rank, answer, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); } } } pmix_status_t prte_data_server_check_range(prte_data_req_t *req, prte_data_object_t *data) { // we automatically accept session and global ranges if (PMIX_RANGE_SESSION == data->range || PMIX_RANGE_GLOBAL == data->range || PMIX_RANGE_UNDEF == data->range) { return PMIX_SUCCESS; } if (PMIX_RANGE_NAMESPACE == data->range) { if (PMIX_CHECK_NSPACE(req->requestor.nspace, data->owner.nspace)) { pmix_output_verbose(10, prte_data_store.output, "%s\tMATCH NSPACES %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), req->requestor.nspace, data->owner.nspace); return PMIX_SUCCESS; } } if (PMIX_RANGE_LOCAL == data->range) { // the sender is the requestor's daemon, so see if // that matches the published data's proxy if (PMIX_CHECK_PROCID(&data->proxy, &req->proxy)) { pmix_output_verbose(10, prte_data_store.output, "%s\tMATCH LOCATION %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIX_NAME_PRINT(&data->proxy), PMIX_NAME_PRINT(&req->proxy)); return PMIX_SUCCESS; } } if (PMIX_RANGE_PROC_LOCAL == data->range) { // the requestor must be the same as the owner if (PMIX_CHECK_PROCID(&data->owner, &req->requestor)) { pmix_output_verbose(10, prte_data_store.output, "%s\tMATCH LOCAL %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIX_NAME_PRINT(&data->owner), PMIX_NAME_PRINT(&req->requestor)); return PMIX_SUCCESS; } } if (PMIX_RANGE_CUSTOM == data->range) { // requestor must be on the list of allowed accessors } if (PMIX_RANGE_RM == data->range) { // the requestor must be from the host - which means // the nspace of the requestor must match that of // the host's server, which is my own if (PMIX_CHECK_NSPACE(req->requestor.nspace, PRTE_PROC_MY_NAME->nspace)) { pmix_output_verbose(10, prte_data_store.output, "%s\tMATCH RM %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), req->requestor.nspace, PRTE_PROC_MY_NAME->nspace); return PMIX_SUCCESS; } } // no matches return PMIX_ERROR; } // CLASS INSTANCE static void construct(prte_data_object_t *ptr) { ptr->index = -1; PMIX_PROC_CONSTRUCT(&ptr->owner); ptr->uid = UINT32_MAX; ptr->range = PMIX_RANGE_SESSION; ptr->persistence = PMIX_PERSIST_SESSION; PMIX_CONSTRUCT(&ptr->info, pmix_list_t); } static void destruct(prte_data_object_t *ptr) { PMIX_LIST_DESTRUCT(&ptr->info); } PMIX_CLASS_INSTANCE(prte_data_object_t, pmix_object_t, construct, destruct); static void rqcon(prte_data_req_t *p) { p->keys = NULL; p->uid = UINT32_MAX; p->range = PMIX_RANGE_UNDEF; } static void rqdes(prte_data_req_t *p) { PMIx_Argv_free(p->keys); } PMIX_CLASS_INSTANCE(prte_data_req_t, pmix_list_item_t, rqcon, rqdes); PMIX_CLASS_INSTANCE(prte_data_cleanup_t, pmix_list_item_t, NULL, NULL); static void dsicon(prte_ds_info_t *p) { PMIX_PROC_CONSTRUCT(&p->source); PMIX_INFO_CONSTRUCT(&p->info); } static void dsides(prte_ds_info_t *p) { PMIX_INFO_DESTRUCT(&p->info); } PMIX_CLASS_INSTANCE(prte_ds_info_t, pmix_list_item_t, dsicon, dsides); prrte-4.1.1~rc2/src/runtime/data_server/ds_unpublish.c0000664000175000017500000001501515203542132023255 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2016 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2025 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/data_server/ds.h" pmix_status_t prte_ds_unpublish(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer) { int32_t count; prte_data_object_t *data; pmix_status_t rc; int k; size_t n, ninfo; uint32_t i; char *str; prte_data_req_t rq; prte_info_item_t *ds1, *ds2; pmix_info_t *info; pmix_output_verbose(1, prte_data_store.output, "%s data server got unpublish from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)); PMIX_CONSTRUCT(&rq, prte_data_req_t); memcpy(&rq.proxy, sender, sizeof(pmix_proc_t)); /* unpack the requestor */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &rq.requestor, &count, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } pmix_output_verbose(1, prte_data_store.output, "%s data server: unpublish data from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIX_NAME_PRINT(&rq.requestor)); /* unpack the number of keys */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &ninfo, &count, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } if (0 == ninfo) { /* they forgot to send us the keys?? */ PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return PMIX_ERR_BAD_PARAM; } /* unpack the keys */ for (n = 0; n < ninfo; n++) { count = 1; rc = PMIx_Data_unpack(NULL, buffer, &str, &count, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&rq); return rc; } PMIx_Argv_append_nosize(&rq.keys, str); free(str); } /* unpack the number of directives, if any */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &ninfo, &count, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } if (0 < ninfo) { PMIX_INFO_CREATE(info, ninfo); count = ninfo; rc = PMIx_Data_unpack(NULL, buffer, info, &count, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); return rc; } /* scan the directives for things we care about */ for (n = 0; n < ninfo; n++) { if (PMIx_Check_key(info[n].key, PMIX_USERID)) { rq.uid = info[n].value.data.uint32; } else if (PMIx_Check_key(info[n].key, PMIX_RANGE)) { rq.range = info[n].value.data.range; } } /* ignore anything else for now */ PMIX_INFO_FREE(info, ninfo); } /* cycle across the provided keys */ for (i = 0; NULL != rq.keys[i]; i++) { /* cycle across the stored data, looking for a match */ for (k = 0; k < prte_data_store.store.size; k++) { data = (prte_data_object_t *) pmix_pointer_array_get_item(&prte_data_store.store, k); if (NULL == data) { continue; } /* can only access data posted by the same user id */ if (rq.uid != data->uid) { continue; } /* can only access data posted by the same process */ if (!PMIX_CHECK_NSPACE(rq.requestor.nspace, data->owner.nspace) || rq.requestor.rank != data->owner.rank) { continue; } /* check the range */ if (PMIX_SUCCESS != prte_data_server_check_range(&rq, data)) { continue; } /* see if we have this key */ PMIX_LIST_FOREACH_SAFE(ds1, ds2, &data->info, prte_info_item_t) { if (PMIx_Check_key(ds1->info.key, rq.keys[i])) { /* found it - remove that item */ pmix_list_remove_item(&data->info, &ds1->super); PMIX_RELEASE(ds1); } } /* if all the data has been removed, then remove the object */ if (0 == pmix_list_get_size(&data->info)) { pmix_pointer_array_set_item(&prte_data_store.store, data->index, NULL); PMIX_RELEASE(data); } } } PMIX_DESTRUCT(&rq); if (PMIX_SUCCESS == rc) { // send back an answer rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } PRTE_RML_SEND(rc, sender->rank, answer, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); } } return rc; } prrte-4.1.1~rc2/src/runtime/data_server/ds_publish.c0000664000175000017500000002771215203542132022721 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2016 Los Alamos National Security, LLC. * All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2025 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_pointer_array.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/data_server/ds.h" pmix_status_t prte_ds_publish(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer) { uint8_t command; int32_t count; prte_data_object_t *data; pmix_data_buffer_t *reply; int rc; size_t ninfo; uint32_t i; bool complete_resolved, found; prte_data_req_t *req, *rqnext; pmix_data_buffer_t pbkt; pmix_byte_object_t pbo; pmix_status_t ret; prte_info_item_t *ds1, *ds2, *ds3; size_t n; pmix_info_t *info; char **cache; pmix_list_t answers; data = PMIX_NEW(prte_data_object_t); memcpy(&data->proxy, sender, sizeof(pmix_proc_t)); /* unpack the publisher */ count = 1; ret = PMIx_Data_unpack(NULL, buffer, &data->owner, &count, PMIX_PROC); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_RELEASE(data); return ret; } pmix_output_verbose(1, prte_data_store.output, "%s data server: publishing data from %s:%d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), data->owner.nspace, data->owner.rank); /* unpack the number of infos and directives they sent */ count = 1; ret = PMIx_Data_unpack(NULL, buffer, &ninfo, &count, PMIX_SIZE); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_RELEASE(data); return ret; } /* if it isn't at least one, then that's an error */ if (1 > ninfo) { ret = PMIX_ERR_BAD_PARAM; PMIX_ERROR_LOG(ret); PMIX_RELEASE(data); rc = PRTE_ERR_UNPACK_FAILURE; return rc; } /* create the space */ PMIX_INFO_CREATE(info, ninfo); /* unpack into it */ count = ninfo; if (PMIX_SUCCESS != (ret = PMIx_Data_unpack(NULL, buffer, info, &count, PMIX_INFO))) { PMIX_ERROR_LOG(ret); PMIX_RELEASE(data); PMIX_INFO_FREE(info, ninfo); rc = PRTE_ERR_UNPACK_FAILURE; return rc; } /* check for directives */ for (n = 0; n < ninfo; n++) { if (PMIx_Check_key(info[n].key, PMIX_RANGE)) { data->range = info[n].value.data.range; } else if (PMIx_Check_key(info[n].key, PMIX_PERSISTENCE)) { data->persistence = info[n].value.data.persist; } else if (PMIx_Check_key(info[n].key, PMIX_USERID)) { data->uid = info[n].value.data.uint32; } else { /* add it to the list of data */ ds1 = PMIX_NEW(prte_info_item_t); PMIX_INFO_XFER(&ds1->info, &info[n]); pmix_list_append(&data->info, &ds1->super); } } // add this data to our store data->index = pmix_pointer_array_add(&prte_data_store.store, data); pmix_output_verbose(1, prte_data_store.output, "%s data server: checking for pending requests", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); /* check for pending requests that match this data */ reply = NULL; rc = PRTE_SUCCESS; PMIX_LIST_FOREACH_SAFE(req, rqnext, &prte_data_store.pending, prte_data_req_t) { if (req->uid != data->uid) { continue; } /* check the range */ if (PMIX_SUCCESS != prte_data_server_check_range(req, data)) { continue; } complete_resolved = false; cache = NULL; PMIX_CONSTRUCT(&answers, pmix_list_t); for (i = 0; NULL != req->keys[i]; i++) { /* cycle thru the data keys for matches */ found = false; PMIX_LIST_FOREACH_SAFE(ds1, ds2, &data->info, prte_info_item_t) { pmix_output_verbose(10, prte_data_store.output, "%s\tCHECKING %s TO %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), ds1->info.key, req->keys[i]); if (PMIx_Check_key(ds1->info.key, req->keys[i])) { pmix_output_verbose(10, prte_data_store.output, "%s data server: packaging return", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); /* track this response */ pmix_output_verbose( 10, prte_data_store.output, "%s data server: adding %s data %s from %s:%d to response", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), ds1->info.key, PMIx_Data_type_string(ds1->info.value.type), data->owner.nspace, data->owner.rank); ds3 = PMIX_NEW(prte_info_item_t); PMIX_INFO_XFER(&ds3->info, &ds1->info); pmix_list_append(&answers, &ds3->super); // if the persistence is "first read", then remove this info if (PMIX_PERSIST_FIRST_READ == data->persistence) { pmix_list_remove_item(&data->info, &ds1->super); PMIX_RELEASE(ds1); } found = true; break; // a key can only occur once } } if (!found) { PMIx_Argv_append_nosize(&cache, req->keys[i]); } } // update the keys to remove all that have been resolved if (0 < PMIx_Argv_count(cache)) { PMIx_Argv_free(req->keys); req->keys = cache; } else { // if no keys are in the cache, then all keys were resolved complete_resolved = true; } n = pmix_list_get_size(&answers); if (0 == n) { PMIX_LIST_DESTRUCT(&answers); continue; } /* send the answers back to the requestor */ pmix_output_verbose(1, prte_data_store.output, "%s data server:publish returning %lu data to %s:%d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (unsigned long)n, req->requestor.nspace, req->requestor.rank); PMIX_DATA_BUFFER_CREATE(reply); /* start with their room number */ rc = PMIx_Data_pack(NULL, reply, &req->room_number, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return rc; } /* we are responding to a lookup cmd */ command = PRTE_PMIX_LOOKUP_CMD; rc = PMIx_Data_pack(NULL, reply, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return rc; } /* if we found all of the requested keys, then indicate so */ if (n == (size_t) PMIx_Argv_count(req->keys)) { rc = PMIX_SUCCESS; } else { rc = PMIX_ERR_PARTIAL_SUCCESS; } /* return the status */ rc = PMIx_Data_pack(NULL, reply, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return rc; } /* pack the rest into a pmix_data_buffer_t */ PMIX_DATA_BUFFER_CONSTRUCT(&pbkt); /* pack the number of returned info's */ if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &n, 1, PMIX_SIZE))) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); rc = PRTE_ERR_PACK_FAILURE; PMIX_DATA_BUFFER_RELEASE(reply); return rc; } /* loop thru and pack the individual responses - this is somewhat less * efficient than packing an info array, but avoids another malloc * operation just to assemble all the return values into a contiguous * array */ while (NULL != (ds3 = (prte_info_item_t *) pmix_list_remove_first(&answers))) { /* pack the data owner */ ret = PMIx_Data_pack(NULL, &pbkt, &data->owner, 1, PMIX_PROC); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); rc = PRTE_ERR_PACK_FAILURE; PMIX_DATA_BUFFER_RELEASE(reply); return rc; } /* pack the data */ ret = PMIx_Data_pack(NULL, &pbkt, &ds3->info, 1, PMIX_INFO); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); rc = PRTE_ERR_PACK_FAILURE; PMIX_DATA_BUFFER_RELEASE(reply); return rc; } } PMIX_LIST_DESTRUCT(&answers); /* unload the pmix buffer */ rc = PMIx_Data_unload(&pbkt, &pbo); /* pack it into our reply */ rc = PMIx_Data_pack(NULL, reply, &pbo, 1, PMIX_BYTE_OBJECT); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(req); return rc; } PRTE_RML_SEND(rc, req->proxy.rank, reply, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); } if (0 == pmix_list_get_size(&data->info)) { // all the data was removed, so we no longer need this entry pmix_pointer_array_set_item(&prte_data_store.store, data->index, NULL); PMIX_RELEASE(data); data = NULL; } if (complete_resolved) { // completely resolved this pending request, so remove it pmix_list_remove_item(&prte_data_store.pending, &req->super); PMIX_RELEASE(req); } if (NULL == data) { break; } } if (PMIX_SUCCESS == rc) { // send back an answer rc = PMIx_Data_pack(NULL, answer, &rc, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } PRTE_RML_SEND(rc, sender->rank, answer, PRTE_RML_TAG_DATA_CLIENT); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); } } return rc; } prrte-4.1.1~rc2/src/runtime/data_server/ds.h0000664000175000017500000000733415203542132021176 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Data server for PRTE */ #ifndef PRTE_DS_INTERNAL_H #define PRTE_DS_INTERNAL_H #include "prte_config.h" #include "types.h" #include "src/pmix/pmix-internal.h" BEGIN_C_DECLS /* define an object to hold data */ typedef struct { /* base object */ pmix_object_t super; /* index of this object in the storage array */ int32_t index; // daemon that sent the data pmix_proc_t proxy; /* process that owns this data - only the * owner can remove it */ pmix_proc_t owner; /* uid of the owner - helps control * access rights */ uint32_t uid; /* characteristics */ pmix_data_range_t range; pmix_persistence_t persistence; /* and the values themselves - we store them as a list * because we may (if persistence is set to "first-read") * remove them upon read */ pmix_list_t info; } prte_data_object_t; PMIX_CLASS_DECLARATION(prte_data_object_t); /* define a request object for delayed answers */ typedef struct { pmix_list_item_t super; pmix_proc_t proxy; pmix_proc_t requestor; int room_number; uint32_t uid; pmix_data_range_t range; char **keys; } prte_data_req_t; PMIX_CLASS_DECLARATION(prte_data_req_t); /* define a container for data object cleanups */ typedef struct { pmix_list_item_t super; prte_data_object_t *data; } prte_data_cleanup_t; /* define a caddy for pointing to pmix_info_t that * are to be included in an answer */ typedef struct { pmix_list_item_t super; pmix_proc_t source; pmix_info_t info; } prte_ds_info_t; PMIX_CLASS_DECLARATION(prte_ds_info_t); typedef struct { pmix_pointer_array_t store; pmix_list_t pending; int output; int verbosity; } prte_data_store_t; extern prte_data_store_t prte_data_store; PRTE_EXPORT pmix_status_t prte_ds_publish(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer); PRTE_EXPORT pmix_status_t prte_ds_lookup(pmix_proc_t *sender, int room_number, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer); PRTE_EXPORT pmix_status_t prte_ds_unpublish(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer); PRTE_EXPORT void prte_ds_purge(pmix_proc_t *sender, pmix_data_buffer_t *buffer, pmix_data_buffer_t *answer); PRTE_EXPORT pmix_status_t prte_data_server_check_range(prte_data_req_t *req, prte_data_object_t *data); END_C_DECLS #endif /* PRTE_DS_INTERNAL_H */ prrte-4.1.1~rc2/src/runtime/data_server/Makefile.am0000664000175000017500000000247515203542132022454 0ustar alastairalastair# -*- makefile -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This makefile.am does not stand on its own - it is included from src/runtime/Makefile.am headers += \ runtime/data_server/prte_data_server.h \ runtime/data_server/ds.h libprrte_la_SOURCES += \ runtime/data_server/ds_main.c \ runtime/data_server/ds_publish.c \ runtime/data_server/ds_lookup.c \ runtime/data_server/ds_unpublish.c \ runtime/data_server/ds_purge.c prrte-4.1.1~rc2/src/runtime/data_server/prte_data_server.h0000664000175000017500000000337315203542132024120 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Data server for PRTE */ #ifndef PRTE_DATA_SERVER_H #define PRTE_DATA_SERVER_H #include "prte_config.h" #include "types.h" #include "src/rml/rml_types.h" #include "src/pmix/pmix-internal.h" BEGIN_C_DECLS #define PRTE_PMIX_PUBLISH_CMD 0x01 #define PRTE_PMIX_LOOKUP_CMD 0x02 #define PRTE_PMIX_UNPUBLISH_CMD 0x03 #define PRTE_PMIX_PURGE_PROC_CMD 0x04 /* provide hooks to startup and finalize the data server */ PRTE_EXPORT int prte_data_server_init(void); PRTE_EXPORT void prte_data_server_finalize(void); /* provide hook for the non-blocking receive */ PRTE_EXPORT void prte_data_server(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); END_C_DECLS #endif /* PRTE_DATA_SERVER_H */ prrte-4.1.1~rc2/src/runtime/runtime_internals.h0000664000175000017500000000251315203542132022025 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Interface into the PRTE Run Time Environment */ #ifndef PRTE_RUNTIME_INTERNALS_H #define PRTE_RUNTIME_INTERNALS_H #include "prte_config.h" BEGIN_C_DECLS /** * Init the PRTE datatype support */ PRTE_EXPORT int prte_dt_init(void); PRTE_EXPORT int prte_preload_default_mca_params(void); END_C_DECLS #endif /* PRTE_RUNTIME_INTERNALS_H */ prrte-4.1.1~rc2/src/runtime/prte_locks.h0000664000175000017500000000320615203542132020430 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Locks to prevent loops inside PRTE */ #ifndef PRTE_LOCKS_H #define PRTE_LOCKS_H #include "prte_config.h" #include "src/threads/pmix_mutex.h" #include "src/threads/pmix_threads.h" BEGIN_C_DECLS /* for everyone */ PRTE_EXPORT extern pmix_mutex_t prte_finalize_lock; /* for HNPs */ PRTE_EXPORT extern pmix_mutex_t prte_abort_inprogress_lock; PRTE_EXPORT extern pmix_mutex_t prte_jobs_complete_lock; PRTE_EXPORT extern pmix_mutex_t prte_quit_lock; PRTE_EXPORT extern pmix_lock_t prte_init_lock; /** * Initialize the locks */ PRTE_EXPORT int prte_locks_init(void); END_C_DECLS #endif /* #ifndef PRTE_LOCKS_H */ prrte-4.1.1~rc2/src/runtime/Makefile.am0000664000175000017500000000376515203542132020160 0ustar alastairalastair# -*- makefile -*- # # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2012 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # Copyright (c) 2023 Jeffrey M. Squyres. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This makefile.am does not stand on its own - it is included from src/Makefile.am EXTRA_DIST += \ runtime/help-prte-runtime.txt headers += \ runtime/runtime.h \ runtime/prte_locks.h \ runtime/prte_globals.h \ runtime/prte_quit.h \ runtime/runtime_internals.h \ runtime/prte_wait.h \ runtime/prte_progress_threads.h libprrte_la_SOURCES += \ runtime/prte_finalize.c \ runtime/prte_init.c \ runtime/prte_locks.c \ runtime/prte_globals.c \ runtime/prte_quit.c \ runtime/data_type_support/prte_dt_copy_fns.c \ runtime/data_type_support/prte_dt_print_fns.c \ runtime/data_type_support/prte_dt_packing_fns.c \ runtime/data_type_support/prte_dt_unpacking_fns.c \ runtime/prte_mca_params.c \ runtime/prte_wait.c \ runtime/prte_progress_threads.c include runtime/data_server/Makefile.am prrte-4.1.1~rc2/src/runtime/prte_wait.c0000664000175000017500000001655715203542132020271 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2008 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_QUEUE_H # include #endif #include #ifdef HAVE_SYS_TIME_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #include #include #include #include #include #ifdef HAVE_SYS_WAIT_H # include #endif #include "src/class/pmix_list.h" #include "src/class/pmix_object.h" #include "src/event/event-internal.h" #include "src/threads/pmix_mutex.h" #include "src/util/pmix_output.h" #include "constants.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/runtime/prte_wait.h" /* Timer Object Declaration */ static void timer_const(prte_timer_t *tm) { tm->ev = prte_event_alloc(); tm->payload = NULL; } static void timer_dest(prte_timer_t *tm) { prte_event_free(tm->ev); } PMIX_CLASS_INSTANCE(prte_timer_t, pmix_object_t, timer_const, timer_dest); static void wccon(prte_wait_tracker_t *p) { p->child = NULL; p->cbfunc = NULL; p->cbdata = NULL; } static void wcdes(prte_wait_tracker_t *p) { if (NULL != p->child) { PMIX_RELEASE(p->child); } } PMIX_CLASS_INSTANCE(prte_wait_tracker_t, pmix_list_item_t, wccon, wcdes); /* Local Variables */ static prte_event_t handler; static pmix_list_t pending_cbs; /* Local Function Prototypes */ static void wait_signal_callback(int fd, short event, void *arg); /* Interface Functions */ void prte_wait_disable(void) { prte_event_del(&handler); } void prte_wait_enable(void) { prte_event_add(&handler, NULL); } int prte_wait_init(void) { PMIX_CONSTRUCT(&pending_cbs, pmix_list_t); prte_event_set(prte_event_base, &handler, SIGCHLD, PRTE_EV_SIGNAL | PRTE_EV_PERSIST, wait_signal_callback, &handler); prte_event_add(&handler, NULL); return PRTE_SUCCESS; } int prte_wait_finalize(void) { prte_event_del(&handler); /* clear out the pending cbs */ PMIX_LIST_DESTRUCT(&pending_cbs); return PRTE_SUCCESS; } /* this function *must* always be called from * within an event in the prte_event_base */ void prte_wait_cb(prte_proc_t *child, prte_wait_cbfunc_t callback, void *data) { prte_wait_tracker_t *t2; if (NULL == child || NULL == callback) { /* bozo protection */ PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return; } /* see if this proc is still alive */ if (!PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_ALIVE)) { if (NULL != callback) { /* already heard this proc is dead, so just do the callback */ t2 = PMIX_NEW(prte_wait_tracker_t); PMIX_RETAIN(child); // protect against race conditions t2->child = child; t2->cbfunc = callback; t2->cbdata = data; prte_event_set(prte_event_base, &t2->ev, -1, PRTE_EV_WRITE, t2->cbfunc, t2); prte_event_active(&t2->ev, PRTE_EV_WRITE, 1); } return; } /* we just override any existing registration */ PMIX_LIST_FOREACH(t2, &pending_cbs, prte_wait_tracker_t) { if (t2->child == child) { t2->cbfunc = callback; t2->cbdata = data; return; } } /* get here if this is a new registration */ t2 = PMIX_NEW(prte_wait_tracker_t); PMIX_RETAIN(child); // protect against race conditions t2->child = child; t2->cbfunc = callback; t2->cbdata = data; pmix_list_append(&pending_cbs, &t2->super); } static void cancel_callback(int fd, short args, void *cbdata) { prte_wait_tracker_t *trk = (prte_wait_tracker_t *) cbdata; prte_wait_tracker_t *t2; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(trk); PMIX_LIST_FOREACH(t2, &pending_cbs, prte_wait_tracker_t) { if (t2->child == trk->child) { pmix_list_remove_item(&pending_cbs, &t2->super); PMIX_RELEASE(t2); PMIX_RELEASE(trk); return; } } PMIX_RELEASE(trk); } void prte_wait_cb_cancel(prte_proc_t *child) { prte_wait_tracker_t *trk; if (NULL == child) { /* bozo protection */ PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return; } /* push this into the event library for handling */ trk = PMIX_NEW(prte_wait_tracker_t); PMIX_RETAIN(child); // protect against race conditions trk->child = child; PRTE_PMIX_THREADSHIFT(trk, prte_event_base, cancel_callback); } /* callback from the event library whenever a SIGCHLD is received */ static void wait_signal_callback(int fd, short event, void *arg) { prte_event_t *signal = (prte_event_t *) arg; int status; pid_t pid; prte_wait_tracker_t *t2; PRTE_HIDE_UNUSED_PARAMS(fd, event); PMIX_ACQUIRE_OBJECT(signal); if (SIGCHLD != PRTE_EVENT_SIGNAL(signal)) { return; } /* we can have multiple children leave but only get one * sigchild callback, so reap all the waitpids until we * don't get anything valid back */ while (1) { pid = waitpid(-1, &status, WNOHANG); if (-1 == pid && EINTR == errno) { /* try it again */ continue; } /* if we got garbage, then nothing we can do */ if (pid <= 0) { return; } /* we are already in an event, so it is safe to access the list */ PMIX_LIST_FOREACH(t2, &pending_cbs, prte_wait_tracker_t) { if (pid == t2->child->pid) { /* found it! */ t2->child->exit_code = status; pmix_list_remove_item(&pending_cbs, &t2->super); if (NULL != t2->cbfunc) { prte_event_set(prte_event_base, &t2->ev, -1, PRTE_EV_WRITE, t2->cbfunc, t2); prte_event_active(&t2->ev, PRTE_EV_WRITE, 1); } else { PMIX_RELEASE(t2); } break; } } } } prrte-4.1.1~rc2/src/runtime/help-prte-runtime.txt0000664000175000017500000000744715203542132022247 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's runtime. # [prte_init:startup:internal-failure] It looks like "prte_init()" failed for some reason. There are many reasons that can cause PRRTE to fail during "prte_init()", some of which are due to configuration or environment problems. This failure appears to be an internal failure — here's some additional information (which may only be relevant to a PRRTE developer): %s failed --> Returned value %s (%d) instead of PRTE_SUCCESS # [prte:session:dir:prohibited] The specified location for the temporary directories required by PRTE is on the list of prohibited locations: Location given: %s Prohibited locations: %s If you believe this is in error, please contact your system administrator to have the list of prohibited locations changed. Otherwise, please identify a different location to be used (use "-h" to see the cmd line option), or simply let the system pick a default location. # [failed-to-uncompress] A compressed message was received that could not be decompressed. This is most likely due to a missing libz library on the receiving node: node: %s Please ensure that the "libz" library is present on all compute nodes. # [bootstrap-not-found] Bootstrap startup was requested, but the required configuration file was not found on at least one node: node: %s file: %s Please ensure the file has been installed in the correct location on every node included in this PRRTE DVM. # [bootstrap-missing-field-name] Bootstrap startup was requested, but the required configuration file on at least one node is missing the field name on an entry: node: %s file: %s line: %s Please correct the line. # [bootstrap-missing-value] Bootstrap startup was requested, but the required configuration file on at least one node is missing a value on an entry: node: %s file: %s line: %s Please correct the line. # [bootstrap-missing-entry] Bootstrap startup was requested, but the required configuration file on at least one node is missing a required entry: node: %s file: %s entry: %s Please provide the missing information. # [bootstrap-bad-nodelist] Bootstrap startup was requested, but we were unable to parse the regex of participating nodes on at least one node: node: %s file: %s regex: %s error: %s Please fix the error and try again. # [prte:session:dir:shared] PRTE has detected that the head of the session directory tree (where scratch files and shared memory backing storage will be placed) resides on a shared file system: Directory: %s File system type: %s For performance reasons, it is strongly recommended that the session directory be located on a local file system. This can be controlled by setting the system temporary directory to be used by PRTE using either the TMPDIR envar or the "prte_tmpdir_base" MCA param. If you need the temporary directory to be different on remote nodes from the local one where %s is running (e.g., when a login node is being employed), then you can set the local temporary directory using the "prte_local_tmpdir_base" MCA param and the one to be used on all other nodes using the "prte_remote_tmpdir_base" param. This is only a warning advisory and your job will continue. You can disable this warning in the future by setting the "prte_silence_shared_fs" MCA param to "1". # [bad-final-order] A PMIx group construct operation was provided with a final order directive that caused some group members to be dropped - i.e., the provided final order does not cover some of the final group members. Please revise your final order directive so that all final group members can be included in the returned final membership. prrte-4.1.1~rc2/src/runtime/prte_wait.h0000664000175000017500000002133715203542132020266 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Interface for waitpid / async notification of child death with the * libevent runtime system. */ #ifndef PRTE_WAIT_H #define PRTE_WAIT_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #include #if HAVE_SYS_TIME_H # include #endif #include "src/event/event-internal.h" #include "src/util/pmix_output.h" #include "src/rml/rml_types.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "types.h" BEGIN_C_DECLS /** typedef for callback function used in \c prte_wait_cb */ typedef void (*prte_wait_cbfunc_t)(int fd, short args, void *cb); /* define a tracker */ typedef struct { pmix_list_item_t super; prte_event_t ev; prte_proc_t *child; prte_wait_cbfunc_t cbfunc; void *cbdata; } prte_wait_tracker_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_wait_tracker_t); /** * Disable / re-Enable SIGCHLD handler * * These functions have to be used after prte_wait_init was called. */ PRTE_EXPORT void prte_wait_enable(void); PRTE_EXPORT void prte_wait_disable(void); /** * Register a callback for process termination * * Register a callback for notification when this process causes a SIGCHLD. * \c waitpid() will have already been called on the process at this * time. */ PRTE_EXPORT void prte_wait_cb(prte_proc_t *proc, prte_wait_cbfunc_t callback, void *data); PRTE_EXPORT void prte_wait_cb_cancel(prte_proc_t *proc); /* In a few places, we need to barrier until something happens * that changes a flag to indicate we can release - e.g., waiting * for a specific message to arrive. If no progress thread is running, * we cycle across prte_progress - however, if a progress thread * is active, then we need to just nanosleep to avoid cross-thread * confusion */ #define PRTE_WAIT_FOR_COMPLETION(flg) \ do { \ pmix_output_verbose(1, prte_progress_thread_debug, \ "%s waiting on progress thread at %s:%d", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__, __LINE__); \ while ((flg)) { \ /* provide a short quiet period so we \ * don't hammer the cpu while waiting \ */ \ struct timespec tp = {0, 100000}; \ nanosleep(&tp, NULL); \ } \ PMIX_ACQUIRE_OBJECT(flg); \ } while (0); /** * In a number of places within the code, we want to setup a timer * to detect when some procedure failed to complete. For example, * when we launch the daemons, we frequently have no way to directly * detect that a daemon failed to launch. Setting a timer allows us * to automatically fail out of the launch if we don't hear from a * daemon in some specified time window. * * Computing the amount of time to wait takes a few lines of code, but * this macro encapsulates those lines along with the timer event * definition just as a convenience. It also centralizes the * necessary checks to ensure that the microsecond field is always * less than 1M since some systems care about that, and to ensure * that the computed wait time doesn't exceed the desired max * wait * * NOTE: the callback function is responsible for releasing the timer * event back to the event pool! */ #define PRTE_DETECT_TIMEOUT(n, deltat, maxwait, cbfunc, cbd) \ do { \ prte_timer_t *_t; \ int _timeout; \ _t = PMIX_NEW(prte_timer_t); \ _t->payload = (cbd); \ prte_event_evtimer_set(prte_event_base, _t->ev, (cbfunc), _t); \ _timeout = (deltat) * (n); \ if ((maxwait) > 0 && _timeout > (maxwait)) { \ _timeout = (maxwait); \ } \ _t->tv.tv_sec = _timeout / 1000000; \ _t->tv.tv_usec = _timeout % 1000000; \ PMIX_OUTPUT_VERBOSE((1, prte_debug_output, "defining timeout: %ld sec %ld usec at %s:%d", \ (long) _t->tv.tv_sec, (long) _t->tv.tv_usec, __FILE__, __LINE__)); \ PMIX_POST_OBJECT(_t); \ prte_event_evtimer_add(_t->ev, &_t->tv); \ } while (0); /** * There are places in the code where we just want to periodically * wakeup to do something, and then go back to sleep again. Setting * a timer allows us to do this * * NOTE: the callback function is responsible for releasing the timer * event back to the event pool when done! Otherwise, the finalize * function will take care of it. */ #define PRTE_TIMER_EVENT(sec, usec, cbfunc) \ do { \ prte_timer_t *tm; \ tm = PMIX_NEW(prte_timer_t); \ prte_event_evtimer_set(prte_event_base, tm->ev, (cbfunc), tm); \ tm->tv.tv_sec = (sec) + (usec) / 1000000; \ tm->tv.tv_usec = (usec) % 1000000; \ PMIX_OUTPUT_VERBOSE((1, prte_debug_output, \ "defining timer event: %ld sec %ld usec at %s:%d", \ (long) tm->tv.tv_sec, (long) tm->tv.tv_usec, __FILE__, __LINE__)); \ PMIX_POST_OBJECT(tm); \ prte_event_evtimer_add(tm->ev, &tm->tv); \ } while (0); /** * \internal * * Initialize the wait system (allocate mutexes, etc.) */ PRTE_EXPORT int prte_wait_init(void); /** * \internal * * Finalize the wait system (deallocate mutexes, etc.) */ PRTE_EXPORT int prte_wait_finalize(void); END_C_DECLS #endif /* #ifndef PRTE_WAIT_H */ prrte-4.1.1~rc2/src/runtime/prte_quit.c0000664000175000017500000004074115203542132020277 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2007-2009 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2021 Amazon.com, Inc. or its affiliates. All Rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_PARAM_H # include #endif #include #include #include #ifdef HAVE_SYS_TYPES_H # include #endif /* HAVE_SYS_TYPES_H */ #ifdef HAVE_SYS_WAIT_H # include #endif /* HAVE_SYS_WAIT_H */ #ifdef HAVE_SYS_TIME_H # include #endif /* HAVE_SYS_TIME_H */ #include "src/mca/errmgr/errmgr.h" #include "src/mca/plm/plm.h" #include "src/mca/state/state.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_output.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_quit.h" #include "src/runtime/runtime.h" /* * Globals */ static int num_aborted = 0; static int num_killed = 0; static int num_failed_start = 0; void prte_quit(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* cleanup */ if (NULL != caddy) { PMIX_RELEASE(caddy); } /* check one-time lock to protect against "bounce" */ if (pmix_mutex_trylock(&prte_quit_lock)) { /* returns 1 if already locked */ return; } /* flag that the event lib should no longer be looped * so we will exit */ prte_event_base_active = false; PMIX_POST_OBJECT(prte_event_base_active); /* break the event loop - this will cause the loop to exit upon completion of any current event */ prte_event_base_loopexit(prte_event_base); } static char *print_aborted_job(prte_job_t *job, prte_app_context_t *approc, prte_proc_t *proc, prte_node_t *node) { char *output = NULL; char *nodename; if (NULL == node) { nodename = "UNKNOWN"; } else { nodename = node->name; } if (PRTE_PROC_STATE_FAILED_TO_START == proc->state || PRTE_PROC_STATE_FAILED_TO_LAUNCH == proc->state) { switch (proc->exit_code) { case PMIX_ERR_SILENT: case PRTE_ERR_SILENT: /* say nothing - it was already reported */ break; case PMIX_ERR_SYS_LIMITS_PIPES: output = pmix_show_help_string("help-prun.txt", "prun:sys-limit-pipe", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_PIPE_SETUP_FAILURE: output = pmix_show_help_string("help-prun.txt", "prun:pipe-setup-failure", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_SYS_LIMITS_CHILDREN: output = pmix_show_help_string("help-prun.txt", "prun:sys-limit-children", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_SYS_LIMITS_FILES: output = pmix_show_help_string("help-prun.txt", "prun:sys-limit-files", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PRTE_ERR_FAILED_GET_TERM_ATTRS: output = pmix_show_help_string("help-prun.txt", "prun:failed-term-attrs", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_JOB_WDIR_NOT_FOUND: output = pmix_show_help_string("help-prun.txt", "prun:wdir-not-found", true, prte_tool_basename, approc->cwd, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_JOB_WDIR_NOT_ACCESSIBLE: output = pmix_show_help_string("help-prun.txt", "prun:wdir-not-accessible", true, prte_tool_basename, approc->cwd, nodename, (unsigned long) proc->name.rank); break; case PMIX_ERR_JOB_EXE_NOT_FOUND: output = pmix_show_help_string("help-prun.txt", "prun:exe-not-found", true, prte_tool_basename, (unsigned long) proc->name.rank, prte_tool_basename, prte_tool_basename, nodename, approc->app); break; case PMIX_ERR_EXE_NOT_ACCESSIBLE: output = pmix_show_help_string("help-prun.txt", "prun:exe-not-accessible", true, prte_tool_basename, approc->app, nodename, (unsigned long) proc->name.rank); break; case PRTE_ERR_MULTIPLE_AFFINITIES: output = pmix_show_help_string("help-prun.txt", "prun:multiple-paffinity-schemes", true, NULL); break; case PRTE_ERR_TOPO_SLOT_LIST_NOT_SUPPORTED: output = pmix_show_help_string("help-prun.txt", "prun:topo-not-supported", true, prte_process_info.nodename, "rankfile containing a slot_list of ", NULL, approc->app); break; case PRTE_ERR_INVALID_NODE_RANK: output = pmix_show_help_string("help-prun.txt", "prun:invalid-node-rank", true); break; case PRTE_ERR_INVALID_LOCAL_RANK: output = pmix_show_help_string("help-prun.txt", "prun:invalid-local-rank", true); break; case PRTE_ERR_NOT_ENOUGH_CORES: output = pmix_show_help_string("help-prun.txt", "prun:not-enough-resources", true, "sockets", nodename, "bind-to-core", approc->app); break; case PRTE_ERR_TOPO_CORE_NOT_SUPPORTED: output = pmix_show_help_string("help-prun.txt", "prun:topo-not-supported", true, nodename, "bind-to-core", "", approc->app); break; case PRTE_ERR_INVALID_PHYS_CPU: output = pmix_show_help_string("help-prun.txt", "prun:invalid-phys-cpu", true); break; case PRTE_ERR_NOT_ENOUGH_SOCKETS: output = pmix_show_help_string("help-prun.txt", "prun:not-enough-resources", true, "sockets", nodename, "bind-to-socket", approc->app); break; case PRTE_ERR_TOPO_SOCKET_NOT_SUPPORTED: output = pmix_show_help_string("help-prun.txt", "prun:topo-not-supported", true, nodename, "bind-to-socket", "", approc->app); break; case PRTE_ERR_MODULE_NOT_FOUND: output = pmix_show_help_string("help-prun.txt", "prun:paffinity-missing-module", true, nodename); break; case PRTE_ERR_SLOT_LIST_RANGE: output = pmix_show_help_string("help-prun.txt", "prun:invalid-slot-list-range", true, nodename, NULL); break; case PRTE_ERR_PIPE_READ_FAILURE: output = pmix_show_help_string("help-prun.txt", "prun:pipe-read-failure", true, prte_tool_basename, nodename, (unsigned long) proc->name.rank); break; case PRTE_ERR_SOCKET_NOT_AVAILABLE: output = pmix_show_help_string("help-prun.txt", "prun:proc-socket-not-avail", true, prte_tool_basename, PRTE_ERROR_NAME(proc->exit_code), nodename, (unsigned long) proc->name.rank); break; default: if (0 != proc->exit_code) { output = pmix_show_help_string("help-prun.txt", "prun:proc-failed-to-start", true, prte_tool_basename, proc->exit_code, PRTE_ERROR_NAME(proc->exit_code), nodename, (unsigned long) proc->name.rank); } else { output = pmix_show_help_string("help-prun.txt", "prun:proc-failed-to-start-no-status", true, prte_tool_basename, nodename); } } return output; } else if (PRTE_PROC_STATE_ABORTED == proc->state || PRTE_PROC_STATE_CALLED_ABORT == proc->state) { output = pmix_show_help_string("help-prun.txt", "prun:proc-ordered-abort", true, prte_tool_basename, (unsigned long) proc->name.rank, (unsigned long) proc->pid, nodename, prte_tool_basename); return output; } else if (PRTE_PROC_STATE_ABORTED_BY_SIG == job->state) { /* aborted by signal */ #ifdef HAVE_STRSIGNAL if (NULL != strsignal(WTERMSIG(proc->exit_code))) { output = pmix_show_help_string("help-prun.txt", "prun:proc-aborted-strsignal", true, prte_tool_basename, (unsigned long) proc->name.rank, (unsigned long) proc->pid, nodename, WTERMSIG(proc->exit_code), strsignal(WTERMSIG(proc->exit_code))); } else { #endif output = pmix_show_help_string("help-prun.txt", "prun:proc-aborted", true, prte_tool_basename, (unsigned long) proc->name.rank, (unsigned long) proc->pid, nodename, WTERMSIG(proc->exit_code)); #ifdef HAVE_STRSIGNAL } #endif return output; } else if (PRTE_PROC_STATE_TERM_WO_SYNC == proc->state) { /* proc exited w/o finalize */ output = pmix_show_help_string("help-prun.txt", "prun:proc-exit-no-sync", true, prte_tool_basename, (unsigned long) proc->name.rank, (unsigned long) proc->pid, nodename, prte_tool_basename, prte_tool_basename); return output; } else if (PRTE_PROC_STATE_COMM_FAILED == proc->state) { output = pmix_show_help_string("help-prun.txt", "prun:proc-comm-failed", true, PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proc->name), nodename); return output; } else if (PRTE_PROC_STATE_SENSOR_BOUND_EXCEEDED == proc->state) { switch (proc->exit_code) { case PRTE_ERR_MEM_LIMIT_EXCEEDED: output = pmix_show_help_string("help-prun.txt", "prun:proc-mem-exceeded", true, PRTE_NAME_PRINT(&proc->name), nodename); break; case PRTE_ERR_PROC_STALLED: output = pmix_show_help_string("help-prun.txt", "prun:proc-stalled", true); break; default: output = pmix_show_help_string("help-prun.txt", "prun:proc-sensor-exceeded", true); } return output; } else if (PRTE_PROC_STATE_HEARTBEAT_FAILED == proc->state) { output = pmix_show_help_string("help-prun.txt", "prun:proc-heartbeat-failed", true, prte_tool_basename, PRTE_NAME_PRINT(&proc->name), nodename); return output; } else if (PRTE_PROC_STATE_TERM_NON_ZERO == proc->state) { if (prte_get_attribute(&job->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, NULL, PMIX_BOOL)) { output = pmix_show_help_string("help-prun.txt", "prun:non-zero-exit", true, prte_tool_basename, PRTE_NAME_PRINT(&proc->name), proc->exit_code); return output; } } /* nothing here */ return NULL; } /* * On abnormal termination - dump the * exit status of the aborted procs. */ static char *dump_job(prte_job_t *job) { int32_t i; prte_proc_t *proc, *pptr; prte_app_context_t *approc; prte_node_t *node; /* cycle through and count the number that were killed or aborted */ for (i = 0; i < job->procs->size; i++) { if (NULL == (pptr = (prte_proc_t *) pmix_pointer_array_get_item(job->procs, i))) { /* array is left-justified - we are done */ break; } if (PRTE_PROC_STATE_FAILED_TO_START == pptr->state || PRTE_PROC_STATE_FAILED_TO_LAUNCH == pptr->state) { ++num_failed_start; } else if (PRTE_PROC_STATE_ABORTED == pptr->state) { ++num_aborted; } else if (PRTE_PROC_STATE_ABORTED_BY_SIG == pptr->state) { ++num_killed; } else if (PRTE_PROC_STATE_SENSOR_BOUND_EXCEEDED == pptr->state) { ++num_killed; } } /* see if there is a guilty party */ proc = NULL; if (!prte_get_attribute(&job->attributes, PRTE_JOB_ABORTED_PROC, (void **) &proc, PMIX_POINTER) || NULL == proc) { return NULL; } approc = (prte_app_context_t *) pmix_pointer_array_get_item(job->apps, proc->app_idx); node = proc->node; return print_aborted_job(job, approc, proc, node); } char *prte_dump_aborted_procs(prte_job_t *jdata) { prte_job_t *job, *launcher; char *output = NULL; /* if we already reported it, then don't do it again */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ERR_REPORTED)) { return NULL; } PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ERR_REPORTED); /* if this job is not a launcher itself, then get the launcher for this job */ if (PMIX_NSPACE_INVALID(jdata->launcher)) { launcher = jdata; } else { launcher = prte_get_job_data_object(jdata->launcher); if (NULL == launcher) { output = strdup("LAUNCHER JOB OBJECT NOT FOUND"); return output; } } /* cycle thru all the children of this launcher to find the * one that caused the error */ /* if this is a non-persistent job, it won't have any child * jobs, so look at it directly */ if (0 == pmix_list_get_size(&launcher->children)) { output = dump_job(jdata); } else { PMIX_LIST_FOREACH(job, &launcher->children, prte_job_t) { output = dump_job(job); if (NULL != output) { break; } } } return output; } prrte-4.1.1~rc2/src/runtime/prte_globals.h0000664000175000017500000006201315203542132020741 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2020 IBM Corporation. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Global params for PRTE */ #ifndef PRTE_RUNTIME_PRTE_GLOBALS_H #define PRTE_RUNTIME_PRTE_GLOBALS_H #include "prte_config.h" #include "types.h" #include #ifdef HAVE_SYS_TIME_H # include #endif #include "src/class/pmix_hash_table.h" #include "src/class/pmix_pointer_array.h" #include "src/class/pmix_ring_buffer.h" #include "src/class/pmix_value_array.h" #include "src/event/event-internal.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/threads/pmix_threads.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/plm/plm_types.h" #include "src/rml/rml_types.h" #include "src/runtime/runtime.h" #include "src/util/attr.h" #include "src/util/pmix_cmd_line.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" BEGIN_C_DECLS PRTE_EXPORT extern int prte_debug_verbosity; /* instantiated in src/runtime/prte_init.c */ PRTE_EXPORT extern char *prte_prohibited_session_dirs; /* instantiated in src/runtime/prte_init.c */ PRTE_EXPORT extern char *prte_job_ident; /* instantiated in src/runtime/prte_globals.c */ PRTE_EXPORT extern bool prte_execute_quiet; /* instantiated in src/runtime/prte_globals.c */ PRTE_EXPORT extern bool prte_report_silent_errors; /* instantiated in src/runtime/prte_globals.c */ PRTE_EXPORT extern bool prte_event_base_active; /* instantiated in src/runtime/prte_init.c */ PRTE_EXPORT extern bool prte_proc_is_bound; /* instantiated in src/runtime/prte_init.c */ PRTE_EXPORT extern int prte_progress_thread_debug; /* instantiated in src/runtime/prte_init.c */ PRTE_EXPORT extern char *prte_tool_basename; // argv[0] of prun or one of its symlinks PRTE_EXPORT extern char *prte_tool_actual; // actual tool executable PRTE_EXPORT extern char *prte_progress_thread_cpus; PRTE_EXPORT extern bool prte_bind_progress_thread_reqd; PRTE_EXPORT extern bool prte_show_launch_progress; PRTE_EXPORT extern bool prte_bootstrap_setup; PRTE_EXPORT extern bool prte_silence_shared_fs; PRTE_EXPORT extern pmix_show_help_file_t prte_show_help_data[]; /** * Global indicating where this process was bound to at launch (will * be NULL if !prte_proc_is_bound) */ PRTE_EXPORT extern hwloc_cpuset_t prte_proc_applied_binding; /* instantiated in src/runtime/prte_init.c */ /* Shortcut for some commonly used names */ #define PRTE_NAME_WILDCARD (&prte_name_wildcard) PRTE_EXPORT extern pmix_proc_t prte_name_wildcard; /** instantiated in src/runtime/prte_init.c */ #define PRTE_NAME_INVALID (&prte_name_invalid) PRTE_EXPORT extern pmix_proc_t prte_name_invalid; /** instantiated in src/runtime/prte_init.c */ #define PRTE_JOBID_WILDCARD (prte_nspace_wildcard) PRTE_EXPORT extern pmix_nspace_t prte_nspace_wildcard; /** instantiated in src/runtime/prte_init.c */ #define PRTE_PROC_MY_NAME (&prte_process_info.myproc) #define PRTE_PROC_MY_PROCID (&prte_process_info.myproc) // backward compatibility synonym /* define a special name that point to my parent (aka the process that spawned me) */ #define PRTE_PROC_MY_PARENT (&prte_process_info.my_parent) /* define a special name that belongs to prte master */ #define PRTE_PROC_MY_HNP (&prte_process_info.my_hnp) /* define some types so we can store the generic * values and still *know* how to convert it for PMIx */ typedef int prte_status_t; typedef uint32_t prte_proc_state_t; // assigned values in src/mca/plm/plm_types.h #define PRTE_PROC_STATE_T PMIX_UINT32 /* define the results values for comparisons so we can change them in only one place */ #define PRTE_VALUE1_GREATER +1 #define PRTE_VALUE2_GREATER -1 #define PRTE_EQUAL 0 /* error manager callback function */ typedef void (*prte_err_cb_fn_t)(pmix_proc_t *proc, prte_proc_state_t state, void *cbdata); /* define an object for timer events */ typedef struct { pmix_object_t super; struct timeval tv; prte_event_t *ev; void *payload; } prte_timer_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_timer_t); PRTE_EXPORT extern int prte_exit_status; /* define some common keys used in PRTE */ #define PRTE_DB_DAEMON_VPID "prte.daemon.vpid" /* State Machine lists */ PRTE_EXPORT extern pmix_list_t prte_job_states; PRTE_EXPORT extern pmix_list_t prte_proc_states; /* a clean output channel without prefix */ PRTE_EXPORT extern int prte_clean_output; #define PRTE_GLOBAL_ARRAY_BLOCK_SIZE 64 #define PRTE_GLOBAL_ARRAY_MAX_SIZE INT_MAX /* define a default error return code for PRTE */ #define PRTE_ERROR_DEFAULT_EXIT_CODE 1 /** * Define a macro for updating the prte_exit_status * The macro provides a convenient way of doing this * so that we can add thread locking at some point * since the prte_exit_status is a global variable. * * Ensure that we do not overwrite the exit status if it has * already been set to some non-zero value. If we don't make * this check, then different parts of the code could overwrite * each other's exit status in the case of abnormal termination. * * For example, if a process aborts, we would record the initial * exit code from the aborted process. However, subsequent processes * will have been aborted by signal as we kill the job. We don't want * the subsequent processes to overwrite the original exit code so * we can tell the user the exit code from the process that caused * the whole thing to happen. */ #define PRTE_UPDATE_EXIT_STATUS(newstatus) \ do { \ if (0 == prte_exit_status && 0 != newstatus) { \ PMIX_OUTPUT_VERBOSE((1, prte_debug_output, "%s:%s(%d) updating exit status to %d", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__, __LINE__, \ newstatus)); \ prte_exit_status = newstatus; \ } \ } while (0); /* sometimes we need to reset the exit status - for example, when we * are restarting a failed process */ #define PRTE_RESET_EXIT_STATUS() \ do { \ PMIX_OUTPUT_VERBOSE((1, prte_debug_output, "%s:%s(%d) reseting exit status", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__, __LINE__)); \ prte_exit_status = 0; \ } while (0); /* define a set of flags to control the launch of a job */ typedef uint16_t prte_job_controls_t; #define PRTE_JOB_CONTROL PRTE_UINT16 /* global type definitions used by RTE - instanced in prte_globals.c */ /************ * Declare this to allow us to use it before fully * defining it - resolves potential circular definition */ struct prte_proc_t; struct prte_job_t; struct prte_job_map_t; struct prte_schizo_base_module_t; /************/ /* define an object for storing node topologies */ typedef struct { pmix_object_t super; int index; hwloc_topology_t topo; } prte_topology_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_topology_t); /* Object for tracking allocations */ typedef struct{ pmix_object_t super; int index; uint32_t session_id; char *user_refid; // PMIX_ALLOC_REQ_ID char *alloc_refid; // PMIX_ALLOC_ID struct timeval timeout; // time limit on session pmix_pointer_array_t *nodes; pmix_pointer_array_t *jobs; pmix_pointer_array_t *children; } prte_session_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_session_t); /** * Information about a specific application to be launched in the RTE. */ typedef struct { /** Parent object */ pmix_object_t super; /** the job this app belongs to */ struct prte_job_t *job; /** Unique index when multiple apps per job */ prte_app_idx_t idx; /** Absolute pathname of argv[0] */ char *app; /** Number of copies of this process that are to be launched */ int32_t num_procs; /** Array of pointers to the proc objects for procs of this app_context * NOTE - not always used */ pmix_pointer_array_t procs; /** State of the app_context */ prte_app_state_t state; /** First MPI rank of this app_context in the job */ pmix_rank_t first_rank; /** Standard argv-style array, including a final NULL pointer */ char **argv; /** Standard environ-style array, including a final NULL pointer */ char **env; /** Current working directory for this app */ char *cwd; /* flags */ prte_app_context_flags_t flags; /* provide a list of attributes for this app_context in place * of having a continually-expanding list of fixed-use values. * This is a list of prte_value_t's, with the intent of providing * flexibility without constantly expanding the memory footprint * every time we want some new (rarely used) option */ pmix_list_t attributes; // store the result of parsing this app's cmd line pmix_cli_result_t cli; } prte_app_context_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_app_context_t); typedef struct { /** Base object so this can be put on a list */ pmix_list_item_t super; /* index of this node object in global array */ int32_t index; /** String node name */ char *name; char *rawname; // name originally given in allocation, if different from name /** aliases */ char **aliases; /* daemon on this node */ struct prte_proc_t *daemon; /* track the unassigned cpus */ hwloc_cpuset_t available; /* cache the cpuset prior to mapping a job for easy reset */ hwloc_cpuset_t jobcache; /** number of procs on this node */ prte_node_rank_t num_procs; /* array of pointers to procs on this node */ pmix_pointer_array_t *procs; /* next node rank on this node */ prte_node_rank_t next_node_rank; /** State of this node */ prte_node_state_t state; /** A "soft" limit on the number of slots available on the node. This will typically correspond to the number of physical CPUs that we have been allocated on this note and would be the "ideal" number of processes for us to launch. */ int32_t slots; /** Slots available for use in the current mapping operation. This * may differ on a per-job basis from the overall allocated slots * thru use of the -host option and possibly other means */ int32_t slots_available; /** How many processes have already been launched, used by one or more jobs on this node. */ int32_t slots_inuse; /** A "hard" limit (if set -- a value of 0 implies no hard limit) on the number of slots that can be allocated on a given node. This is for some environments (e.g. grid) there may be fixed limits on the number of slots that can be used. This value also could have been a boolean - but we may want to allow the hard limit be different than the soft limit - in other words allow the node to be oversubscribed up to a specified limit. For example, if we have two processors, we may want to allow up to four processes but no more. */ int32_t slots_max; /* system topology for this node */ prte_topology_t *topology; // topology diff from referenced topology hwloc_topology_diff_t topodiff; /* flags */ prte_node_flags_t flags; /* list of prte_attribute_t */ pmix_list_t attributes; } prte_node_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_node_t); typedef struct { /** Base object so this can be put on a list */ pmix_list_item_t super; /* record the exit status for this job */ int exit_code; /* personality for this job */ char **personality; struct prte_schizo_base_module_t *schizo; /* jobid for this job */ pmix_nspace_t nspace; // session directory for this job char *session_dir; int index; // index in the job array where this is stored /* offset to the total number of procs so shared memory * components can potentially connect to any spawned jobs*/ pmix_rank_t offset; /* session this job is running in */ prte_session_t *session; /* app_context array for this job */ pmix_pointer_array_t *apps; /* number of app_contexts in the array */ prte_app_idx_t num_apps; /* rank desiring stdin - for now, either one rank, all ranks * (wildcard), or none (invalid) */ pmix_rank_t stdin_target; /* total slots allocated to this job */ int32_t total_slots_alloc; /* number of procs in this job */ pmix_rank_t num_procs; /* array of pointers to procs in this job */ pmix_pointer_array_t *procs; /* map of the job */ struct prte_job_map_t *map; /* bookmark for where we are in mapping - this * indicates the node where we stopped */ prte_node_t *bookmark; /* state of the overall job */ prte_job_state_t state; /* number of procs mapped */ pmix_rank_t num_mapped; /* number of procs launched */ pmix_rank_t num_launched; /* number of procs reporting contact info */ pmix_rank_t num_reported; /* number of procs terminated */ pmix_rank_t num_terminated; /* number of daemons reported launched so we can track progress */ pmix_rank_t num_daemons_reported; /* number of procs ready for debug */ pmix_rank_t num_ready_for_debug; /* originator of a dynamic spawn */ pmix_proc_t originator; /* number of local procs */ pmix_rank_t num_local_procs; /* flags */ prte_job_flags_t flags; /* attributes */ pmix_list_t attributes; /* launch msg buffer */ pmix_data_buffer_t launch_msg; /* track children of this job */ pmix_list_t children; /* track the launcher of these jobs */ pmix_nspace_t launcher; /* track the number of stack traces recv'd */ uint32_t ntraces; char **traces; // store the result of parsing this app's cmd line pmix_cli_result_t cli; } prte_job_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_job_t); struct prte_proc_t { /** Base object so this can be put on a list */ pmix_list_item_t super; /* process name */ pmix_proc_t name; /* the vpid of my parent - the daemon vpid for an app * or the vpid of the parent in the routing tree of * a daemon */ pmix_rank_t parent; /* pid */ pid_t pid; /* local rank amongst my peers on the node * where this is running - this value is * needed by MPI procs so that the lowest * rank on a node can perform certain fns - * e.g., open an sm backing file */ prte_local_rank_t local_rank; /* local rank on the node across all procs * and jobs known to this HNP - this is * needed so that procs can do things like * know which static IP port to use */ prte_node_rank_t node_rank; /* rank of this proc within its app context - this * will just equal its vpid for single app_context * applications */ int32_t app_rank; /* rank of this proc amongst its peers within the * NUMA region to which it is bound */ prte_local_rank_t numa_rank; /* Last state used to trigger the errmgr for this proc */ prte_proc_state_t last_errmgr_state; /* process state */ prte_proc_state_t state; /* exit code */ prte_exit_code_t exit_code; /* the app_context that generated this proc */ prte_app_idx_t app_idx; /* pointer to the node where this proc is executing */ prte_node_t *node; /* pointer to the object on that node where the * proc is mapped */ hwloc_obj_t obj; /* cpuset where the proc is bound */ char *cpuset; /* RML contact info */ char *rml_uri; /* some boolean flags */ prte_proc_flags_t flags; /* list of prte_value_t attributes */ pmix_list_t attributes; }; typedef struct prte_proc_t prte_proc_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_proc_t); /** Get session object */ PRTE_EXPORT prte_session_t *prte_get_session_object(const uint32_t session_id); PRTE_EXPORT prte_session_t *prte_get_session_object_from_id(const char *id); PRTE_EXPORT prte_session_t *prte_get_session_object_from_refid(const char *refid); PRTE_EXPORT int prte_set_session_object(prte_session_t *session); PRTE_EXPORT bool prte_sessions_related(prte_session_t *session1, prte_session_t *session2); /** * Get a job data object * We cannot just reference a job data object with its jobid as * the jobid is no longer an index into the array. This change * was necessitated by modification of the jobid to include * an mpirun-unique qualifer to eliminate any global name * service */ PRTE_EXPORT prte_job_t *prte_get_job_data_object(const pmix_nspace_t job); /** * Set a job data object - returns an error if it cannot add the object * to the array */ PRTE_EXPORT int prte_set_job_data_object(prte_job_t *jdata); /** Pack/unpack a job object */ PRTE_EXPORT int prte_job_pack(pmix_data_buffer_t *bkt, prte_job_t *job); PRTE_EXPORT int prte_job_unpack(pmix_data_buffer_t *bkt, prte_job_t **job); PRTE_EXPORT int prte_job_copy(prte_job_t **dest, prte_job_t *src); PRTE_EXPORT void prte_job_print(char **output, prte_job_t *jdata); /** Pack/unpack an app-context */ PRTE_EXPORT int prte_app_pack(pmix_data_buffer_t *bkt, prte_app_context_t *app); PRTE_EXPORT int prte_app_unpack(pmix_data_buffer_t *bkt, prte_app_context_t **app); PRTE_EXPORT int prte_app_copy(prte_app_context_t **dest, prte_app_context_t *src); PRTE_EXPORT void prte_app_print(char **output, prte_job_t *jdata, prte_app_context_t *src); /** Pack/unpack a proc*/ PRTE_EXPORT int prte_proc_pack(pmix_data_buffer_t *bkt, prte_proc_t *proc); PRTE_EXPORT int prte_proc_unpack(pmix_data_buffer_t *bkt, prte_proc_t **proc); PRTE_EXPORT int prte_proc_copy(prte_proc_t **dest, prte_proc_t *src); PRTE_EXPORT void prte_proc_print(char **output, prte_job_t *jdata, prte_proc_t *src); /** Pack/unpack a job map */ PRTE_EXPORT int prte_map_pack(pmix_data_buffer_t *bkt, struct prte_job_map_t *map); PRTE_EXPORT int prte_map_unpack(pmix_data_buffer_t *bkt, struct prte_job_map_t **map); PRTE_EXPORT int prte_map_copy(struct prte_job_map_t **dest, struct prte_job_map_t *src); PRTE_EXPORT void prte_map_print(char **output, prte_job_t *jdata); PRTE_EXPORT int prte_node_pack(pmix_data_buffer_t *bkt, prte_node_t *node); PRTE_EXPORT int prte_node_unpack(pmix_data_buffer_t *bkt, prte_node_t **node); PRTE_EXPORT int prte_node_copy(prte_node_t **dest, prte_node_t *src); PRTE_EXPORT void prte_node_print(char **output, prte_job_t *jdata, prte_node_t *src); /** * Get a proc data object */ PRTE_EXPORT prte_proc_t *prte_get_proc_object(const pmix_proc_t *proc); /** * Get the daemon vpid hosting a given proc */ PRTE_EXPORT pmix_rank_t prte_get_proc_daemon_vpid(const pmix_proc_t *proc); /* Get the hostname of a proc */ PRTE_EXPORT char *prte_get_proc_hostname(const pmix_proc_t *proc); /* get the node rank of a proc */ PRTE_EXPORT prte_node_rank_t prte_get_proc_node_rank(const pmix_proc_t *proc); /* check to see if two nodes match */ PRTE_EXPORT prte_node_t* prte_node_match(pmix_list_t *nodes, const char *name); PRTE_EXPORT bool prte_nptr_match(prte_node_t *n1, prte_node_t *n2); PRTE_EXPORT bool prte_quickmatch(prte_node_t *nd, char *name); /* global variables used by RTE - instanced in prte_globals.c */ PRTE_EXPORT extern bool prte_debug_daemons_flag; PRTE_EXPORT extern bool prte_debug_daemons_file_flag; PRTE_EXPORT extern bool prte_leave_session_attached; PRTE_EXPORT extern char *prte_data_server_uri; PRTE_EXPORT extern bool prte_dvm_ready; PRTE_EXPORT extern pmix_pointer_array_t *prte_cache; PRTE_EXPORT extern bool prte_persistent; PRTE_EXPORT extern bool prte_allow_run_as_root; PRTE_EXPORT extern bool prte_fwd_environment; PRTE_EXPORT extern bool prte_xml_output; /* PRTE OOB port flags */ PRTE_EXPORT extern bool prte_static_ports; PRTE_EXPORT extern char *prte_oob_static_ports; /* nodename flags */ PRTE_EXPORT extern bool prte_keep_fqdn_hostnames; PRTE_EXPORT extern bool prte_have_fqdn_allocation; PRTE_EXPORT extern bool prte_show_resolved_nodenames; PRTE_EXPORT extern int prte_hostname_cutoff; PRTE_EXPORT extern bool prte_do_not_resolve; /* debug flags */ PRTE_EXPORT extern pmix_rank_t prted_debug_failure; PRTE_EXPORT extern int prted_debug_failure_delay; PRTE_EXPORT extern bool prte_never_launched; PRTE_EXPORT extern bool prte_devel_level_output; PRTE_EXPORT extern bool prte_display_topo_with_map; PRTE_EXPORT extern char **prte_launch_environ; PRTE_EXPORT extern bool prte_hnp_is_allocated; PRTE_EXPORT extern bool prte_allocation_required; PRTE_EXPORT extern bool prte_managed_allocation; PRTE_EXPORT extern char *prte_set_slots; PRTE_EXPORT extern bool prte_set_slots_override; PRTE_EXPORT extern bool prte_hnp_connected; PRTE_EXPORT extern bool prte_nidmap_communicated; PRTE_EXPORT extern bool prte_node_info_communicated; /* launch agents */ PRTE_EXPORT extern char *prte_launch_agent; PRTE_EXPORT extern char **prted_cmd_line; /* exit flags */ PRTE_EXPORT extern bool prte_abnormal_term_ordered; PRTE_EXPORT extern bool prte_routing_is_enabled; PRTE_EXPORT extern bool prte_dvm_abort_ordered; PRTE_EXPORT extern bool prte_prteds_term_ordered; PRTE_EXPORT extern bool prte_allowed_exit_without_sync; PRTE_EXPORT extern int prte_timeout_usec_per_proc; PRTE_EXPORT extern float prte_max_timeout; PRTE_EXPORT extern prte_timer_t *prte_mpiexec_timeout; /* global arrays for data storage */ PRTE_EXPORT extern pmix_pointer_array_t *prte_sessions; PRTE_EXPORT extern pmix_pointer_array_t *prte_job_data; PRTE_EXPORT extern pmix_pointer_array_t *prte_node_pool; PRTE_EXPORT extern pmix_pointer_array_t *prte_node_topologies; PRTE_EXPORT extern pmix_pointer_array_t *prte_local_children; PRTE_EXPORT extern pmix_rank_t prte_total_procs; PRTE_EXPORT extern char *prte_base_compute_node_sig; PRTE_EXPORT extern bool prte_homo_nodes; /* IOF controls */ /* generate new xterm windows to display output from specified ranks */ PRTE_EXPORT extern char *prte_xterm; /* whether or not to report launch progress */ PRTE_EXPORT extern bool prte_report_launch_progress; /* allocation specification */ PRTE_EXPORT extern char *prte_default_hostfile; PRTE_EXPORT extern bool prte_default_hostfile_given; PRTE_EXPORT extern int prte_num_allocated_nodes; PRTE_EXPORT extern char *prte_default_dash_host; PRTE_EXPORT extern prte_session_t *prte_default_session; /* tool communication controls */ PRTE_EXPORT extern bool prte_report_events; PRTE_EXPORT extern char *prte_report_events_uri; /* exit status reporting */ PRTE_EXPORT extern bool prte_report_child_jobs_separately; PRTE_EXPORT extern struct timeval prte_child_time_to_exit; /* length of stat history to keep */ PRTE_EXPORT extern int prte_stat_history_size; /* maximum size of virtual machine - used to subdivide allocation */ PRTE_EXPORT extern int prte_max_vm_size; /* binding directives for daemons to restrict them * to certain cores */ PRTE_EXPORT extern char *prte_daemon_cores; /* Max time to wait for stack straces to return */ PRTE_EXPORT extern int prte_stack_trace_wait_timeout; /* whether or not hwloc shmem support is available */ PRTE_EXPORT extern bool prte_hwloc_shmem_available; extern char *prte_signal_string; extern char *prte_stacktrace_output_filename; extern char *prte_net_private_ipv4; extern char *prte_set_max_sys_limits; extern char *prte_if_include; extern char *prte_if_exclude; extern char *prte_param_files; extern char* prte_override_param_file; extern bool prte_suppress_override_warning; #if PRTE_PICKY_COMPILERS #define PRTE_HIDE_UNUSED_PARAMS(...) \ do { \ int __x = 3; \ prte_hide_unused_params(__x, __VA_ARGS__); \ } while(0) PMIX_EXPORT void prte_hide_unused_params(int x, ...); #else #define PRTE_HIDE_UNUSED_PARAMS(...) #endif END_C_DECLS #endif /* PRTE_RUNTIME_PRTE_GLOBALS_H */ prrte-4.1.1~rc2/src/mca/0000775000175000017500000000000015203542132015166 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/common/0000775000175000017500000000000015203542132016456 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/common/Makefile.am0000664000175000017500000000224715203542132020517 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # Note that this file must exist, even though it is empty (there is no # "base" directory for the common framework). autogen.pl and # prte_mca.m4 assume that every framework has a top-level Makefile.am. # We *could* adjust the framework glue code to exclude "common" from # this requirement, but it's just a lot easier to have an empty # Makefile.am here. prrte-4.1.1~rc2/src/mca/rmaps/0000775000175000017500000000000015203542132016310 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/ppr/0000775000175000017500000000000015203542132017111 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/ppr/owner.txt0000664000175000017500000000024415203542132021004 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/rmaps/ppr/rmaps_ppr_component.c0000664000175000017500000000457015203542132023350 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/util/pmix_show_help.h" #include "rmaps_ppr.h" #include "src/mca/rmaps/base/base.h" /* * Local functions */ static int prte_rmaps_ppr_open(void); static int prte_rmaps_ppr_close(void); static int prte_rmaps_ppr_query(pmix_mca_base_module_t **module, int *priority); static int prte_rmaps_ppr_register(void); prte_rmaps_base_component_t prte_mca_rmaps_ppr_component = { PRTE_RMAPS_BASE_VERSION_4_0_0, .pmix_mca_component_name = "ppr", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PRTE_RELEASE_VERSION), .pmix_mca_open_component = prte_rmaps_ppr_open, .pmix_mca_close_component = prte_rmaps_ppr_close, .pmix_mca_query_component = prte_rmaps_ppr_query, .pmix_mca_register_component_params = prte_rmaps_ppr_register, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, rmaps, ppr) static int my_priority; static int prte_rmaps_ppr_open(void) { return PRTE_SUCCESS; } static int prte_rmaps_ppr_query(pmix_mca_base_module_t **module, int *priority) { *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_rmaps_ppr_module; return PRTE_SUCCESS; } /** * Close all subsystems. */ static int prte_rmaps_ppr_close(void) { return PRTE_SUCCESS; } static int prte_rmaps_ppr_register(void) { my_priority = 90; (void) pmix_mca_base_component_var_register(&prte_mca_rmaps_ppr_component, "priority", "Priority of the ppr rmaps component", PMIX_MCA_BASE_VAR_TYPE_INT, &my_priority); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/ppr/help-prte-rmaps-ppr.txt0000664000175000017500000000322115203542132023467 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2024-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # [invalid-ppr] An invalid value was given for the number of processes per resource (ppr) to be mapped on each node: PPR: %s The specification must be a comma-separated list containing combinations of number, followed by a colon, followed by the resource type. For example, a value of "1:package" indicates that one process is to be mapped onto each package. Values are supported for "hwthread", "core", "package", "l1cache", "l2cache", "l3cache", and "node" . Note that enough characters must be provided to clearly specify the desired resource (e.g., "l1" for "l1cache"). # [unrecognized-ppr-option] An unrecognized value was given for the number of processes per resource (ppr) to be mapped on each node: Value: %s PPR: %s The specification must be a number, followed by a colon, followed by the resource type. For example, a value of "1:slot" indicates that anything over one process per slot is to be considered oversubscribed. Only values for "hwthread", "core", "package", "l1cache", "l2cache", "l3cache", and "node" are allowed. Note that enough characters must be provided to clearly specify the desired resource (e.g., "l1" for "l1cache"). # [ppr-too-many-procs] Your job has requested more processes than the ppr for this topology can support: App: %s Number of procs: %d Procs mapped: %d Total number of procs: %d PPR: %s Please revise the conflict and try again. prrte-4.1.1~rc2/src/mca/rmaps/ppr/rmaps_ppr.h0000664000175000017500000000135015203542132021264 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_RMAPS_PPR_H #define PRTE_RMAPS_PPR_H #include "prte_config.h" #include "src/hwloc/hwloc-internal.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS PRTE_MODULE_EXPORT extern prte_rmaps_base_component_t prte_mca_rmaps_ppr_component; extern prte_rmaps_base_module_t prte_rmaps_ppr_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/ppr/Makefile.am0000664000175000017500000000222115203542132021142 0ustar alastairalastair# # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-prte-rmaps-ppr.txt sources = \ rmaps_ppr.c \ rmaps_ppr.h \ rmaps_ppr_component.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_rmaps_ppr_DSO component_noinst = component_install = prte_mca_rmaps_ppr.la else component_noinst = libprtemca_rmaps_ppr.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_rmaps_ppr_la_SOURCES = $(sources) prte_mca_rmaps_ppr_la_LDFLAGS = -module -avoid-version prte_mca_rmaps_ppr_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_rmaps_ppr_la_SOURCES =$(sources) libprtemca_rmaps_ppr_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/rmaps/ppr/rmaps_ppr.c0000664000175000017500000004270215203542132021265 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/util/pmix_show_help.h" #include "rmaps_ppr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" static int ppr_mapper(prte_job_t *jdata, prte_rmaps_options_t *options); prte_rmaps_base_module_t prte_rmaps_ppr_module = { .map_job = ppr_mapper }; static int ppr_mapper(prte_job_t *jdata, prte_rmaps_options_t *options) { int rc = PRTE_SUCCESS, j, idx, ncpus; prte_proc_t *proc; pmix_mca_base_component_t *c = &prte_mca_rmaps_ppr_component; prte_node_t *node, *nd; prte_app_context_t *app; int nprocs_mapped; prte_mapping_policy_t mapping = 0; prte_ranking_policy_t ranking; hwloc_obj_t obj; unsigned int nobjs, i; pmix_list_t node_list; int32_t num_slots; char *jobppr = NULL; bool initial_map = true; prte_binding_policy_t savebind = options->bind; uint16_t ppn, pes, *ppnptr, *pesptr; uint16_t jobppn, jobpes; /* only handle initial launch of loadbalanced * or NPERxxx jobs - allow restarting of failed apps */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:ppr: job %s being restarted - ppr cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (NULL != jdata->map->req_mapper && 0 != strcasecmp(jdata->map->req_mapper, c->pmix_mca_component_name)) { /* a mapper has been specified, and it isn't me */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:ppr: job %s not using ppr mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PPR, (void **) &jobppr, PMIX_STRING) || NULL == jobppr || PRTE_MAPPING_PPR != PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) { /* not for us */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:ppr: job %s not using ppr mapper PPR %s policy %s", PRTE_JOBID_PRINT(jdata->nspace), (NULL == jobppr) ? "NULL" : jobppr, (PRTE_MAPPING_PPR == PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) ? "PPRSET" : "PPR NOTSET"); if (NULL != jobppr) { free(jobppr); } return PRTE_ERR_TAKE_NEXT_OPTION; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:ppr: mapping job %s with ppr %s", PRTE_JOBID_PRINT(jdata->nspace), jobppr); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->pmix_mca_component_name); ranking = PRTE_RANK_BY_SLOT; if (HWLOC_OBJ_MACHINE == options->maptype) { mapping = PRTE_MAPPING_BYNODE; ranking = PRTE_RANK_BY_NODE; } else if (HWLOC_OBJ_PACKAGE == options->maptype) { mapping = PRTE_MAPPING_BYPACKAGE; } else if (HWLOC_OBJ_NUMANODE== options->maptype) { mapping = PRTE_MAPPING_BYNUMA; } else if (HWLOC_OBJ_L1CACHE == options->maptype) { mapping = PRTE_MAPPING_BYL1CACHE; } else if (HWLOC_OBJ_L2CACHE == options->maptype) { mapping = PRTE_MAPPING_BYL2CACHE; } else if (HWLOC_OBJ_L3CACHE == options->maptype) { mapping = PRTE_MAPPING_BYL3CACHE; } else if (HWLOC_OBJ_CORE == options->maptype) { mapping = PRTE_MAPPING_BYCORE; } else if (HWLOC_OBJ_PU == options->maptype) { mapping = PRTE_MAPPING_BYHWTHREAD; } /* record the results */ PRTE_SET_MAPPING_POLICY(jdata->map->mapping, mapping); if (!PRTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) { PRTE_SET_RANKING_POLICY(jdata->map->ranking, ranking); } options->map = PRTE_GET_MAPPING_POLICY(jdata->map->mapping); options->rank = PRTE_GET_RANKING_POLICY(jdata->map->ranking); if (PRTE_RANK_BY_SPAN == options->rank || PRTE_RANK_BY_FILL == options->rank) { if (options->map < PRTE_MAPPING_BYNUMA || options->map > PRTE_MAPPING_BYHWTHREAD) { pmix_show_help("help-prte-rmaps-base.txt", "must-map-by-obj", true, prte_rmaps_base_print_mapping(options->map), prte_rmaps_base_print_ranking(options->rank)); free(jobppr); return PRTE_ERR_SILENT; } } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:ppr: job %s assigned policy %s:%s", PRTE_JOBID_PRINT(jdata->nspace), prte_rmaps_base_print_mapping(options->map), prte_rmaps_base_print_ranking(options->rank)); // cache job-level values jobppn = options->pprn; jobpes = options->cpus_per_rank; ppnptr = &ppn; pesptr = &pes; /* cycle thru the apps */ for (idx = 0; idx < jdata->apps->size; idx++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, idx); if (NULL == app) { continue; } options->total_nobjs = 0; if (prte_get_attribute(&app->attributes, PRTE_APP_PPR, (void**)&ppnptr, PMIX_UINT16)) { options->pprn = ppn; } else { options->pprn = jobppn; } if (prte_get_attribute(&app->attributes, PRTE_APP_PES_PER_PROC, (void**)&pesptr, PMIX_UINT16)) { options->cpus_per_rank = pes; } else { options->cpus_per_rank = jobpes; } /* get the available nodes */ PMIX_CONSTRUCT(&node_list, pmix_list_t); rc = prte_rmaps_base_get_target_nodes(&node_list, &num_slots, jdata, app, jdata->map->mapping, initial_map, false, false); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* flag that all subsequent requests should not reset the node->mapped flag */ initial_map = false; if (0 == app->num_procs) { // compute the number of procs if (HWLOC_OBJ_MACHINE == options->maptype) { app->num_procs = options->pprn * pmix_list_get_size(&node_list); } else if (HWLOC_OBJ_PACKAGE == options->maptype) { /* add in #packages for each node */ PMIX_LIST_FOREACH (node, &node_list, prte_node_t) { nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE); app->num_procs += options->pprn * nobjs; } } else if (HWLOC_OBJ_NUMANODE== options->maptype) { /* add in #numa for each node */ PMIX_LIST_FOREACH (node, &node_list, prte_node_t) { nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_NUMANODE); app->num_procs += options->pprn * nobjs; } } else if (HWLOC_OBJ_L1CACHE == options->maptype || HWLOC_OBJ_L2CACHE == options->maptype || HWLOC_OBJ_L3CACHE == options->maptype) { /* add in #cache for each node */ PMIX_LIST_FOREACH (node, &node_list, prte_node_t) { nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->maptype); app->num_procs += options->pprn * nobjs; } } else if (HWLOC_OBJ_CORE == options->maptype) { /* add in #cores for each node */ PMIX_LIST_FOREACH (node, &node_list, prte_node_t) { nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_CORE); app->num_procs += options->pprn * nobjs; } } else if (HWLOC_OBJ_PU == options->maptype) { /* add in #hwt for each node */ PMIX_LIST_FOREACH (node, &node_list, prte_node_t) { nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PU); app->num_procs += options->pprn * nobjs; } } } /* check to see if we can map all the procs */ if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL) && num_slots < (int) app->num_procs) { if (!options->oversubscribe) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); rc = PRTE_ERR_SILENT; goto error; } else { if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { jdata->map->binding = PRTE_BIND_TO_NONE; options->bind = PRTE_BIND_TO_NONE; } } } /* cycle across the nodes */ nprocs_mapped = 0; PMIX_LIST_FOREACH_SAFE(node, nd, &node_list, prte_node_t) { options->nobjs = 0; prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto error; } if (!options->donotlaunch) { rc = prte_rmaps_base_check_support(jdata, node, options); if (PRTE_SUCCESS != rc) { goto error; } } if (HWLOC_OBJ_MACHINE == options->maptype) { options->nprocs = options->pprn; /* if there are not enough slots to support the required * number of procs, and they didn't specify oversubscribe, * then we cannot use this node */ if (options->nprocs > node->slots_available && !options->oversubscribe) { // skip this node continue; } /* if the number of procs times the number of pes/proc * is greater than the number of CPUs * on this node, but the number of procs is less or equal * to the number of slots, * then we are not oversubscribed but we are overloaded. If * the user didn't specify a required binding, then we set * the binding policy to do-not-bind for this node */ ncpus = prte_rmaps_base_get_ncpus(node, NULL, options); if ((options->nprocs * options->cpus_per_rank) > ncpus && options->nprocs <= node->slots_available && !PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { options->bind = PRTE_BIND_TO_NONE; jdata->map->binding = PRTE_BIND_TO_NONE; } /* check availability and set the target cpuset - this * also computes the nprocs to be assigned capped by * the number of available binding targets */ if (!prte_rmaps_base_check_avail(jdata, app, node, &node_list, NULL, options)) { options->bind = savebind; continue; } for (j = 0; j < options->pprn && nprocs_mapped < app->num_procs; j++) { proc = prte_rmaps_base_setup_proc(jdata, idx, node, NULL, options); if (NULL == proc) { rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); return rc; } PMIX_RELEASE(proc); } } else { /* get the number of resources on this node */ nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->maptype); if (0 == nobjs) { continue; } options->nprocs = options->pprn * nobjs; /* if there are not enough slots to support the required * number of procs, and they didn't specify oversubscribe, * then we cannot use this node */ if (options->nprocs > node->slots_available && !options->oversubscribe) { // skip this node continue; } /* if the number of procs times the number of pes/proc * is greater than the number of CPUs * on this node, but the number of procs is less or equal * to the number of slots, * then we are not oversubscribed but we are overloaded. If * the user didn't specify a required binding, then we set * the binding policy to do-not-bind for this node */ ncpus = prte_rmaps_base_get_ncpus(node, NULL, options); if ((options->nprocs * options->cpus_per_rank) > ncpus && options->nprocs <= node->slots_available && !PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { options->bind = PRTE_BIND_TO_NONE; jdata->map->binding = PRTE_BIND_TO_NONE; } /* map the specified number of procs to each such resource on this node */ for (i = 0; i < nobjs && nprocs_mapped < app->num_procs; i++) { obj = prte_hwloc_base_get_obj_by_type(node->topology->topo, options->maptype, i); // are there enough cpus on this obj to meet the request? if (!prte_rmaps_base_check_avail(jdata, app, node, &node_list, obj, options)) { continue; } for (j=0; j < options->pprn && nprocs_mapped < app->num_procs; j++) { proc = prte_rmaps_base_setup_proc(jdata, idx, node, obj, options); if (NULL == proc) { rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); return rc; } PMIX_RELEASE(proc); } } } options->bind = savebind; /* if we haven't mapped all the procs, continue on to the * next node */ if (nprocs_mapped == app->num_procs) { break; } } if (0 == app->num_procs) { app->num_procs = nprocs_mapped; } if (nprocs_mapped < app->num_procs) { /* couldn't map them all */ pmix_show_help("help-prte-rmaps-ppr.txt", "ppr-too-many-procs", true, app->app, app->num_procs, nprocs_mapped, options->nprocs, jobppr); rc = PRTE_ERR_SILENT; goto error; } jdata->num_procs += app->num_procs; PMIX_LIST_DESTRUCT(&node_list); } free(jobppr); /* calculate the ranks for this app */ rc = prte_rmaps_base_compute_vpids(jdata, options); return rc; error: PMIX_LIST_DESTRUCT(&node_list); free(jobppr); return rc; } prrte-4.1.1~rc2/src/mca/rmaps/lsf/0000775000175000017500000000000015203542132017074 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/lsf/owner.txt0000664000175000017500000000024315203542132020766 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: IBM status: unmaintained prrte-4.1.1~rc2/src/mca/rmaps/lsf/rmaps_lsf.h0000664000175000017500000000367315203542132021244 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2022 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Mapping */ #ifndef PRTE_RMAPS_RF_H #define PRTE_RMAPS_RF_H #include "prte_config.h" #include "src/class/pmix_object.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS #define RMAPS_LSF_MAX_SLOTS 64 struct prte_rmaps_rf_component_t { prte_rmaps_base_component_t super; char *slot_list; }; typedef struct prte_rmaps_rf_component_t prte_rmaps_rf_component_t; PRTE_MODULE_EXPORT extern prte_rmaps_rf_component_t prte_mca_rmaps_lsf_component; extern prte_rmaps_base_module_t prte_rmaps_lsf_module; typedef struct cpu_package_t cpu_package_t; typedef struct { pmix_object_t super; char *node_name; char slot_list[RMAPS_LSF_MAX_SLOTS]; } prte_rmaps_lsf_map_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_rmaps_lsf_map_t); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/lsf/rmaps_lsf_component.c0000664000175000017500000000563015203542132023314 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_base.h" #include "src/util/pmix_show_help.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" #include "src/mca/rmaps/lsf/rmaps_lsf.h" /* * Local functions */ static int lsf_query(pmix_mca_base_module_t **module, int *priority); prte_rmaps_rf_component_t prte_mca_rmaps_lsf_component = { .super = { PRTE_RMAPS_BASE_VERSION_4_0_0, .pmix_mca_component_name = "lsf", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_query_component = lsf_query, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, rmaps, lsf) static int lsf_query(pmix_mca_base_module_t **module, int *priority) { /* * Set the component priority to the highest: * - If we are in an LSF environment with affinity information (LSB_AFFINITY_HOSTFILE) * the LSF ras component will have forced this mapper */ *priority = 100; *module = (pmix_mca_base_module_t *) &prte_rmaps_lsf_module; return PRTE_SUCCESS; } static void map_construct(prte_rmaps_lsf_map_t *ptr) { ptr->node_name = NULL; memset(ptr->slot_list, (char) 0x00, RMAPS_LSF_MAX_SLOTS); } static void map_destruct(prte_rmaps_lsf_map_t *ptr) { if (NULL != ptr->node_name) free(ptr->node_name); } PMIX_CLASS_INSTANCE(prte_rmaps_lsf_map_t, pmix_object_t, map_construct, map_destruct); prrte-4.1.1~rc2/src/mca/rmaps/lsf/help-rmaps_lsf.txt0000664000175000017500000000446715203542132022564 0ustar alastairalastair# Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2018-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for rankle utilities. # [bad-syntax] Error, invalid syntax in the rankfile (%s) syntax must be the following rank i=host_i slot=string Examples of proper syntax include: rank 1=host1 slot=1:0,1 rank 0=host2 slot=0:* rank 2=host4 slot=1-2 rank 3=host3 slot=0:1;1:0-2 # [missing-rank] A rank is missing its location specification: Rank: %d Rank file: %s All processes must have their location specified in the rank file. Either add an entry to the file, or provide a default slot_list to use for any unspecified ranks. # [missing-cpu] While parsing the rankfile, %s encountered a line that specified a non-existent CPU: Slots: %s Available CPUs: %s Please correct the line and try again. # [lsf-affinity-file-not-found] The affinity file provided in LSB_AFFINITY_HOSTFILE could not be found: File: %s We cannot continue. # [rmaps:proc-slots-overloaded] A process attempted to bind to resources already allocated on a node to a different process. Process : %s Node : %s Process Requested CPU Set : %s Node Available CPU Set : %s Overlapping CPU Set : %s If this is intentional then you must pass the "overload-allowed" qualifier to the --bind-to option. --bind-to :overload-allowed # [resource-not-found] The specified LSF affinity file contained a node (%s) that is not in your allocation. We therefore cannot map a process rank to it. Please check your allocation and affinity file to ensure the latter only contains allocated nodes. # [bad-host] The rankfile that was used claimed that a host was either not allocated or oversubscribed its slots. Please review your rank-slot assignments and your host allocation to ensure a proper match. Also, some systems may require using full hostnames, such as "host1.example.com" (instead of just plain "host1"). Host: %s prrte-4.1.1~rc2/src/mca/rmaps/lsf/configure.m40000664000175000017500000000317415203542132021324 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ras_lsf_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_rmaps_lsf_CONFIG],[ AC_CONFIG_FILES([src/mca/rmaps/lsf/Makefile]) PRTE_CHECK_LSF([rmaps_lsf], [rmaps_lsf_good=1], [rmaps_lsf_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$rmaps_lsf_good" = "1" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([rmaps_lsf_CPPFLAGS]) AC_SUBST([rmaps_lsf_DFLAGS]) AC_SUBST([rmaps_lsf_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/rmaps/lsf/rmaps_lsf.c0000664000175000017500000005617715203542132021246 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2022 IBM Corporation. All rights reserved. * * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_fd.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/proc_info.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" #include "src/mca/rmaps/lsf/rmaps_lsf.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_show_help.h" static int lsf_map(prte_job_t *jdata, prte_rmaps_options_t *options); prte_rmaps_base_module_t prte_rmaps_lsf_module = { .map_job = lsf_map }; static int file_parse(const char *); /* * Local variable */ static pmix_pointer_array_t rankmap; static int num_ranks = 0; /* * Create a rank_file mapping for the job. */ static int lsf_map(prte_job_t *jdata, prte_rmaps_options_t *options) { prte_app_context_t *app = NULL; int32_t i, k; pmix_list_t node_list; prte_node_t *node, *nd, *root_node; pmix_rank_t rank, vpid_start; int32_t num_slots; prte_rmaps_lsf_map_t *rfmap; int32_t relative_index, tmp_cnt; int rc; prte_proc_t *proc; pmix_mca_base_component_t *c = &prte_mca_rmaps_lsf_component.super; char *slots = NULL; bool initial_map = true; char *affinity_file = NULL; hwloc_cpuset_t proc_bitmap, bitmap; char *cpu_bitmap; char *avail_bitmap = NULL; char *overlap_bitmap = NULL; bool physical; /* only handle initial launch of rf job */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: job %s being restarted - lsf cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } affinity_file = getenv("LSB_AFFINITY_HOSTFILE"); if (NULL == affinity_file) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: affinity file not given in environment"); return PRTE_ERR_TAKE_NEXT_OPTION; } if (PRTE_MAPPING_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) { // user gave a mapping directive, so it cannot be us pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: mapping directive given - skipping lsf"); return PRTE_ERR_TAKE_NEXT_OPTION; } if (options->ordered) { /* NOT FOR US */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: job %s binding order requested - lsf cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } physical = prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_PHYSICAL_CPUS, NULL, PMIX_BOOL); /* LSF provides its info as hwthreads, so set the hwthread-as-cpus flag */ prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); options->use_hwthreads = true; /* don't override something provided by the user, but default to bind-to hwthread */ if (!PRTE_BINDING_POLICY_IS_SET(prte_hwloc_default_binding_policy)) { PRTE_SET_BINDING_POLICY(prte_hwloc_default_binding_policy, PRTE_BIND_TO_HWTHREAD); } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: mapping job %s", PRTE_JOBID_PRINT(jdata->nspace)); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->pmix_mca_component_name); /* setup the node list */ PMIX_CONSTRUCT(&node_list, pmix_list_t); /* pickup the first app - there must be at least one */ app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, 0); if (NULL == app) { rc = PRTE_ERR_SILENT; goto error; } /* start at the beginning... */ vpid_start = 0; jdata->num_procs = 0; PMIX_CONSTRUCT(&rankmap, pmix_pointer_array_t); rc = pmix_pointer_array_init(&rankmap, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != rc) { PMIX_DESTRUCT(&rankmap); return PRTE_ERROR; } /* parse the affinity, storing its results in the rankmap */ if (PRTE_SUCCESS != (rc = file_parse(affinity_file))) { rc = PRTE_ERR_SILENT; goto error; } /* cycle through the app_contexts, mapping them sequentially */ for (i = 0; i < jdata->apps->size; i++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } /* for each app_context, we have to get the list of nodes that it can * use since that can now be modified with a hostfile and/or -host * option */ rc = prte_rmaps_base_get_target_nodes(&node_list, &num_slots, jdata, app, options->map, initial_map, false, true); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* flag that all subsequent requests should not reset the node->mapped flag */ initial_map = false; /* set the number of procs to the number of entries in that affinity_file */ if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_COMPUTED)) { app->num_procs = num_ranks; if (0 == app->num_procs) { pmix_show_help("help-rmaps_lsf.txt", "bad-syntax", true, affinity_file); rc = PRTE_ERR_SILENT; goto error; } } for (k = 0; k < app->num_procs; k++) { rank = vpid_start + k; /* get the rankfile entry for this rank */ rfmap = (prte_rmaps_lsf_map_t *) pmix_pointer_array_get_item(&rankmap, rank); if (NULL == rfmap) { /* if this job was given a slot-list, then use it */ if (NULL != options->cpuset) { slots = options->cpuset; } else if (NULL != prte_hwloc_default_cpu_list) { /* if we were give a default slot-list, then use it */ slots = prte_hwloc_default_cpu_list; } else { /* all ranks must be specified */ pmix_show_help("help-rmaps_lsf.txt", "missing-rank", true, rank, affinity_file); rc = PRTE_ERR_SILENT; goto error; } /* take the next node off of the available list */ node = NULL; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { /* if adding one to this node would oversubscribe it, then try * the next one */ if (nd->slots <= (int) nd->num_procs) { continue; } /* take this one */ node = nd; break; } if (NULL == node) { /* all would be oversubscribed, so take the least loaded one */ k = (int32_t) UINT32_MAX; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { if (nd->num_procs < (pmix_rank_t) k) { k = nd->num_procs; node = nd; } } } /* if we still have nothing, then something is very wrong */ if (NULL == node) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } } else { if (0 == strlen(rfmap->slot_list)) { slots = NULL; } else { slots = rfmap->slot_list; } /* find the node where this proc was assigned */ node = NULL; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { if (NULL != rfmap->node_name && prte_quickmatch(nd, rfmap->node_name)) { node = nd; break; } else if (NULL != rfmap->node_name && (('+' == rfmap->node_name[0]) && (('n' == rfmap->node_name[1]) || ('N' == rfmap->node_name[1])))) { relative_index = atoi(strtok(rfmap->node_name, "+n")); if (relative_index >= (int) pmix_list_get_size(&node_list) || (0 > relative_index)) { pmix_show_help("help-rmaps_lsf.txt", "bad-index", true, rfmap->node_name); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } root_node = (prte_node_t *) pmix_list_get_first(&node_list); for (tmp_cnt = 0; tmp_cnt < relative_index; tmp_cnt++) { root_node = (prte_node_t *) pmix_list_get_next(root_node); } node = root_node; break; } } } if (NULL == node) { pmix_show_help("help-rmaps_lsf.txt", "resource-not-found", true, rfmap->node_name); rc = PRTE_ERR_SILENT; goto error; } if (!options->donotlaunch) { rc = prte_rmaps_base_check_support(jdata, node, options); if (PRTE_SUCCESS != rc) { return rc; } } prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto error; } if (!prte_rmaps_base_check_avail(jdata, app, node, &node_list, NULL, options)) { pmix_show_help("help-rmaps_lsf.txt", "bad-host", true, rfmap->node_name); rc = PRTE_ERR_SILENT; goto error; } /* check if we are oversubscribed */ rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_SUCCESS != rc) { goto error; } options->map = PRTE_MAPPING_BYUSER; proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } /* set the vpid */ proc->name.rank = rank; /* Assign this process to these slots * Allow for overload in cases where different ranks are assigned to * the same PU, but it must be requested by the user. */ if (NULL != slots && (PRTE_BIND_TO_NONE != PRTE_GET_BINDING_POLICY(jdata->map->binding) || options->overload) ) { if (NULL == node->topology || NULL == node->topology->topo) { // Not allowed - for rank-file, we must have the topology pmix_show_help("help-prte-rmaps-base.txt", "rmaps:no-topology", true, node->name); rc = PRTE_ERR_SILENT; goto error; } proc_bitmap = hwloc_bitmap_alloc(); /* parse the slot_list to find the package and core */ rc = prte_hwloc_base_cpu_list_parse(slots, node->topology->topo, options->use_hwthreads, proc_bitmap); if (PRTE_ERR_NOT_FOUND == rc) { char *tmp = prte_hwloc_base_cset2str(hwloc_topology_get_allowed_cpuset(node->topology->topo), false, physical, node->topology->topo); pmix_show_help("help-rmaps_lsf.txt", "missing-cpu", true, prte_tool_basename, slots, tmp); free(tmp); rc = PRTE_ERR_SILENT; hwloc_bitmap_free(proc_bitmap); goto error; } else if (PRTE_ERROR == rc) { pmix_show_help("help-rmaps_lsf.txt", "bad-syntax", true, affinity_file); rc = PRTE_ERR_SILENT; hwloc_bitmap_free(proc_bitmap); goto error; } else if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); hwloc_bitmap_free(proc_bitmap); goto error; } /* note that we cannot set the proc locale to any specific object * as the slot list may have assigned it to more than one - so * leave that field NULL */ /* set the proc to the specified map */ hwloc_bitmap_list_asprintf(&cpu_bitmap, proc_bitmap); proc->cpuset = strdup(cpu_bitmap); pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rank_file: convert slots from <%s> to <%s>", slots, cpu_bitmap); /* Check to see if these slots are available on this node */ if (!hwloc_bitmap_isincluded(proc_bitmap, node->available) && !options->overload) { bitmap = hwloc_bitmap_alloc(); hwloc_bitmap_list_asprintf(&avail_bitmap, node->available); hwloc_bitmap_andnot(bitmap, proc_bitmap, node->available); hwloc_bitmap_list_asprintf(&overlap_bitmap, bitmap); pmix_show_help("help-rmaps_lsf.txt", "rmaps:proc-slots-overloaded", true, PRTE_NAME_PRINT(&proc->name), node->name, proc->cpuset, avail_bitmap, overlap_bitmap); hwloc_bitmap_free(bitmap); hwloc_bitmap_free(proc_bitmap); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } /* Mark these slots as taken on this node */ hwloc_bitmap_andnot(node->available, node->available, proc_bitmap); /* cleanup */ free(cpu_bitmap); hwloc_bitmap_free(proc_bitmap); } /* insert the proc into the proper place */ PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(proc); goto error; } jdata->num_procs++; PMIX_RELEASE(proc); } /* update the starting point */ vpid_start += app->num_procs; /* cleanup the node list - it can differ from one app_context * to another, so we have to get it every time */ PMIX_LIST_DESTRUCT(&node_list); PMIX_CONSTRUCT(&node_list, pmix_list_t); } PMIX_LIST_DESTRUCT(&node_list); /* cleanup the rankmap */ for (i = 0; i < rankmap.size; i++) { if (NULL != (rfmap = pmix_pointer_array_get_item(&rankmap, i))) { PMIX_RELEASE(rfmap); } } PMIX_DESTRUCT(&rankmap); /* compute local/app ranks */ rc = prte_rmaps_base_compute_vpids(jdata, options); return rc; error: PMIX_LIST_DESTRUCT(&node_list); return rc; } static int file_parse(const char *affinity_file) { int rc = PRTE_SUCCESS; int i, j; prte_rmaps_lsf_map_t *rfmap = NULL; pmix_pointer_array_t *assigned_ranks_array; struct stat buf; FILE *fp; char *hstname, *membind_opt; char *sep, *eptr, **cpus, *ptr; prte_node_t *nptr, *node; hwloc_obj_t obj; /* keep track of rank assignments */ assigned_ranks_array = PMIX_NEW(pmix_pointer_array_t); rc = pmix_pointer_array_init(assigned_ranks_array, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(assigned_ranks_array); return PRTE_ERROR; } /* check to see if the file is empty - if it is, * then affinity wasn't actually set for this job */ if (0 != stat(affinity_file, &buf)) { pmix_show_help("help-rmaps_lsf.txt", "lsf-affinity-file-not-found", true, affinity_file); return PRTE_ERR_SILENT; } if (0 == buf.st_size) { /* no affinity, so just return */ return PRTE_SUCCESS; } /* open the file */ fp = fopen(affinity_file, "r"); if (NULL == fp) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } while (NULL != (hstname = pmix_getline(fp))) { if (0 == strlen(hstname)) { free(hstname); /* blank line - ignore */ continue; } if ('#' == hstname[0]) { free(hstname); /* Comment line - ignore */ continue; } if (NULL != (sep = strchr(hstname, ' '))) { *sep = '\0'; sep++; /* remove any trailing space */ eptr = sep + strlen(sep) - 1; while (eptr > sep && isspace(*eptr)) { eptr--; } *(eptr + 1) = 0; /* * If the submitted LSF job has memory binding related resource requirement, after * the cpu id list there are memory binding options. * * The following is the format of LSB_AFFINITY_HOSTFILE file: * * Host1 0,1,2,3 0 2 * Host1 4,5,6,7 1 2 * * Each line includes: host_name, cpu_id_list, NUMA_node_id_list, and memory_policy. * In this fix we will drop the last two sections (NUMA_node_id_list and memory_policy) * of each line and keep them in 'membind_opt' for future use. */ if (NULL != (membind_opt = strchr(sep, ' '))) { *membind_opt = '\0'; membind_opt++; } } // Convert the Physical CPU set from LSF to a Hwloc logical CPU set pmix_output_verbose(20, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: (lsf) Convert Physical CPUSET from <%s>", sep); // if we are not keeping fqdn, remove the domain name here if (!prte_keep_fqdn_hostnames) { if (!pmix_net_isaddr(hstname) && NULL != (ptr = strchr(hstname, '.'))) { *ptr = '\0'; } } // find the named host nptr = NULL; for (j = 0; j < prte_node_pool->size; j++) { node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, j); if (NULL == node) { continue; } if (prte_quickmatch(node, hstname)) { nptr = node; break; } } if (NULL == nptr) { /* wasn't found - that is an error */ pmix_show_help("help-rmaps_lsf.txt", "resource-not-found", true, hstname); fclose(fp); free(hstname); return PRTE_ERROR; } if (NULL != sep) { cpus = PMIx_Argv_split(sep, ','); for(i = 0; NULL != cpus[i]; ++i) { // get the specified object obj = hwloc_get_pu_obj_by_os_index(nptr->topology->topo, strtol(cpus[i], NULL, 10)) ; if (NULL == obj) { PMIx_Argv_free(cpus); fclose(fp); free(hstname); return PRTE_ERROR; } free(cpus[i]); // 10 max number of digits in an int cpus[i] = (char*)malloc(sizeof(char) * 10); snprintf(cpus[i], 10, "%d", obj->logical_index); } sep = PMIx_Argv_join(cpus, ','); PMIx_Argv_free(cpus); pmix_output_verbose(20, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: (lsf) Convert Physical CPUSET to <%s>", sep); } rfmap = PMIX_NEW(prte_rmaps_lsf_map_t); rfmap->node_name = hstname; if (NULL != sep) { snprintf(rfmap->slot_list, RMAPS_LSF_MAX_SLOTS, "%s", sep); } pmix_pointer_array_set_item(&rankmap, num_ranks, rfmap); num_ranks++; // keep track of number of provided ranks pmix_output_verbose(20, prte_rmaps_base_framework.framework_output, "mca:rmaps:lsf: Adding node %s cpus %s", rfmap->node_name, rfmap->slot_list); } fclose(fp); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/lsf/Makefile.am0000664000175000017500000000372215203542132021134 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008 Voltaire. All rights reserved # # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-rmaps_lsf.txt sources = \ rmaps_lsf.c \ rmaps_lsf.h \ rmaps_lsf_component.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_rmaps_lsf_DSO component_noinst = component_install = prte_mca_rmaps_lsf.la else component_noinst = libprtemca_rmaps_lsf.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_rmaps_lsf_la_SOURCES = $(sources) prte_mca_rmaps_lsf_la_LDFLAGS = -module -avoid-version prte_mca_rmaps_lsf_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_rmaps_lsf_la_SOURCES =$(sources) libprtemca_rmaps_lsf_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/rmaps/rmaps.h0000664000175000017500000000641615203542132017612 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The PRTE Resource MAPping Subsystem (RMAPS) * * The resource mapping subsystem is responsible for mapping processes * to specific nodes/cpus within a given job. In many systems, this * functionality will not be supported - the system will map processes * wherever it chooses and does not allow the user to specify the * mapping. RMAPS components, therefore, provide services for those * systems that do permit such mappings. * * RMAPS checks the MCA parameters to see if a mapping algorithm has * been specified. If the user selected a mapping algorithm, the * indicated RMAPS component will take information from the registry * to determine the number of applications/processes to be run, and * the identified resources that have been allocated to this job. The * selected RMAP component will then assign processes to resources * according to its algorithm, with the results stored on the * appropriate job segment - the assigned nodename for each process is * stored in that respective process' container on the segment. * */ #ifndef PRTE_MCA_RMAPS_H #define PRTE_MCA_RMAPS_H #include "prte_config.h" #include "types.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/mca/rmaps/rmaps_types.h" BEGIN_C_DECLS /* * rmaps module functions */ /** * RMAPS module functions - these are not accessible to the outside world, * but are defined here by convention */ /* map a job - used by the HNP to compute the #procs on each node. * This is passed to the backend daemons as a regex which they * use to create an prte_job_map_t for the job */ typedef int (*prte_rmaps_base_module_map_fn_t)(prte_job_t *jdata, prte_rmaps_options_t *options); /* * rmaps module version 4.0.0 */ struct prte_rmaps_base_module_4_0_0_t { /** Mapping function pointer */ prte_rmaps_base_module_map_fn_t map_job; }; /** Convenience typedef */ typedef struct prte_rmaps_base_module_4_0_0_t prte_rmaps_base_module_4_0_0_t; /** Convenience typedef */ typedef prte_rmaps_base_module_4_0_0_t prte_rmaps_base_module_t; /* * rmaps component */ /** * rmaps component version 3.0.0 */ typedef pmix_mca_base_component_t prte_rmaps_base_component_t; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/seq/0000775000175000017500000000000015203542132017100 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/seq/owner.txt0000664000175000017500000000024215203542132020771 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: IBM status: maintenance prrte-4.1.1~rc2/src/mca/rmaps/seq/rmaps_seq_component.c0000664000175000017500000000601415203542132023321 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "rmaps_seq.h" #include "src/mca/rmaps/rmaps.h" /* * Local functions */ static int prte_rmaps_seq_register(void); static int prte_rmaps_seq_open(void); static int prte_rmaps_seq_close(void); static int prte_rmaps_seq_query(pmix_mca_base_module_t **module, int *priority); static int my_priority; prte_rmaps_base_component_t prte_mca_rmaps_seq_component = { PRTE_RMAPS_BASE_VERSION_4_0_0, .pmix_mca_component_name = "seq", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_open_component = prte_rmaps_seq_open, .pmix_mca_close_component = prte_rmaps_seq_close, .pmix_mca_query_component = prte_rmaps_seq_query, .pmix_mca_register_component_params = prte_rmaps_seq_register, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, rmaps, seq) /** * component register/open/close/init function */ static int prte_rmaps_seq_register(void) { my_priority = 60; (void) pmix_mca_base_component_var_register(&prte_mca_rmaps_seq_component, "priority", "Priority of the seq rmaps component", PMIX_MCA_BASE_VAR_TYPE_INT, &my_priority); return PRTE_SUCCESS; } static int prte_rmaps_seq_open(void) { return PRTE_SUCCESS; } static int prte_rmaps_seq_query(pmix_mca_base_module_t **module, int *priority) { *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_rmaps_seq_module; return PRTE_SUCCESS; } /** * Close all subsystems. */ static int prte_rmaps_seq_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/seq/rmaps_seq.h0000664000175000017500000000261215203542132021244 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Mapping */ #ifndef PRTE_RMAPS_SEQ_H #define PRTE_RMAPS_SEQ_H #include "prte_config.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS /** * RMGR Component */ PRTE_MODULE_EXPORT extern prte_rmaps_base_component_t prte_mca_rmaps_seq_component; extern prte_rmaps_base_module_t prte_rmaps_seq_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/seq/help-prte-rmaps-seq.txt0000664000175000017500000000370115203542132023450 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2018-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's prun. # [prte-rmaps-seq:resource-not-found] The specified sequence file contained a node (%s) that is not in your allocation. We therefore cannot map a process rank to it. Please check your allocation and sequence file to ensure the latter only contains allocated nodes. # [seq:not-enough-resources] A sequential map was requested, but not enough node entries were given to support the requested number of processes: Num procs: %d Num nodes: %d The sequential mapper requires that there be a node entry for every process in the job, and it processes the provided file using each node entry to identify the node where that numbered rank is to be placed. Note that you can provide a separate sequence file (different from a hostfile that specifies the number of slots on each node) for use by the sequential mapper in its assignments by using the appropriate modifier: Example: --map-by seq:file=myseqfile # [proc-failed-to-map] A process could not be mapped: Hostname: %s App: %s Could not continue. prrte-4.1.1~rc2/src/mca/rmaps/seq/rmaps_seq.c0000664000175000017500000004256215203542132021247 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/runtime/prte_globals.h" #include "src/util/dash_host/dash_host.h" #include "src/util/hostfile/hostfile.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "rmaps_seq.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" static int prte_rmaps_seq_map(prte_job_t *jdata, prte_rmaps_options_t *options); /* define the module */ prte_rmaps_base_module_t prte_rmaps_seq_module = { .map_job = prte_rmaps_seq_map }; /* local object for tracking rank locations */ typedef struct { pmix_list_item_t super; char *hostname; char *cpuset; } seq_node_t; static void sn_con(seq_node_t *p) { p->hostname = NULL; p->cpuset = NULL; } static void sn_des(seq_node_t *p) { if (NULL != p->hostname) { free(p->hostname); p->hostname = NULL; } if (NULL != p->cpuset) { free(p->cpuset); p->cpuset = NULL; } } PMIX_CLASS_INSTANCE(seq_node_t, pmix_list_item_t, sn_con, sn_des); static int process_file(char *path, pmix_list_t *list); /* * Sequentially map the ranks according to the placement in the * specified hostfile */ static int prte_rmaps_seq_map(prte_job_t *jdata, prte_rmaps_options_t *options) { prte_job_map_t *map; prte_app_context_t *app; int i, n; int32_t j; prte_node_t *node, *nd; seq_node_t *sq, *save = NULL, *seq, *seq2; pmix_rank_t vpid, apprank; int32_t num_nodes; int rc; pmix_list_t default_seq_list; pmix_list_t node_list, *seq_list, sq_list; prte_proc_t *proc; pmix_mca_base_component_t *c = &prte_mca_rmaps_seq_component; char *hosts = NULL; bool match; PMIX_OUTPUT_VERBOSE((1, prte_rmaps_base_framework.framework_output, "%s rmaps:seq called on job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); /* this mapper can only handle initial launch * when seq mapping is desired - allow * restarting of failed apps */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: job %s is being restarted - seq cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (NULL != jdata->map->req_mapper) { if (0 != strcasecmp(jdata->map->req_mapper, c->pmix_mca_component_name)) { /* a mapper has been specified, and it isn't me */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: job %s not using sequential mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } } if (PRTE_MAPPING_SEQ != PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) { /* I don't know how to do these - defer */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: job %s not using seq mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: mapping job %s", PRTE_JOBID_PRINT(jdata->nspace)); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->pmix_mca_component_name); /* convenience def */ map = jdata->map; /* if there is a default hostfile, go and get its ordered list of nodes */ PMIX_CONSTRUCT(&default_seq_list, pmix_list_t); if (NULL != prte_default_hostfile) { rc = process_file(prte_default_hostfile, &default_seq_list); if (PRTE_SUCCESS != rc) { PMIX_LIST_DESTRUCT(&default_seq_list); return rc; } } /* start at the beginning... */ vpid = 0; jdata->num_procs = 0; /* cycle through the app_contexts, mapping them sequentially */ for (i = 0; i < jdata->apps->size; i++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } apprank = 0; /* specified seq file trumps all */ hosts = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, (void **) &hosts, PMIX_STRING)) { if (NULL == hosts) { rc = PRTE_ERR_NOT_FOUND; goto error; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: using hostfile %s nodes on app %s", hosts, app->app); PMIX_CONSTRUCT(&sq_list, pmix_list_t); rc = process_file(hosts, &sq_list); free(hosts); hosts = NULL; if (PRTE_SUCCESS != rc) { PMIX_LIST_DESTRUCT(&sq_list); goto error; } seq_list = &sq_list; goto process; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING)) { if (NULL == hosts) { rc = PRTE_ERR_NOT_FOUND; goto error; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: using dash-host nodes on app %s", app->app); PMIX_CONSTRUCT(&node_list, pmix_list_t); /* dash host entries cannot specify cpusets, so used the std function to retrieve the * list */ rc = prte_util_get_ordered_dash_host_list(&node_list, hosts); free(hosts); hosts = NULL; if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* transfer the list to a seq_node_t list */ PMIX_CONSTRUCT(&sq_list, pmix_list_t); while (NULL != (nd = (prte_node_t *) pmix_list_remove_first(&node_list))) { sq = PMIX_NEW(seq_node_t); sq->hostname = strdup(nd->name); pmix_list_append(&sq_list, &sq->super); PMIX_RELEASE(nd); } PMIX_DESTRUCT(&node_list); seq_list = &sq_list; goto process; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_HOSTFILE, (void **) &hosts, PMIX_STRING)) { if (NULL == hosts) { rc = PRTE_ERR_NOT_FOUND; goto error; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: using hostfile %s nodes on app %s", hosts, app->app); PMIX_CONSTRUCT(&sq_list, pmix_list_t); rc = process_file(hosts, &sq_list); free(hosts); hosts = NULL; if (PRTE_SUCCESS != rc) { PMIX_LIST_DESTRUCT(&sq_list); goto error; } seq_list = &sq_list; goto process; } /* if we get here, then there were no app-level directives, so try to use the * default hostfile list */ if (0 < pmix_list_get_size(&default_seq_list)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: using default hostfile nodes on app %s", app->app); seq_list = &default_seq_list; } else { /* can't do anything - no nodes available! */ pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-available-resources", true); rc = PRTE_ERR_SILENT; goto error; } process: /* check for nolocal and remove the head node, if required */ if (PRTE_GET_MAPPING_DIRECTIVE(map->mapping) & PRTE_MAPPING_NO_USE_LOCAL) { PMIX_LIST_FOREACH_SAFE(seq, seq2, seq_list, seq_node_t) { /* need to check ifislocal because the name in the * hostfile may not have been FQDN, while name returned * by gethostname may have been (or vice versa) */ if (prte_check_host_is_local(seq->hostname)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: removing head node %s", seq->hostname); pmix_list_remove_item(seq_list, &seq->super); PMIX_RELEASE(seq); /* "un-retain" it */ } } } if (NULL == seq_list || 0 == (num_nodes = (int32_t) pmix_list_get_size(seq_list))) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-available-resources", true); rc = PRTE_ERR_SILENT; goto error; } /* set #procs to the number of entries */ if (0 == app->num_procs) { app->num_procs = num_nodes; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: setting num procs to %s for app %s", PRTE_VPID_PRINT(app->num_procs), app->app); } else if (num_nodes < app->num_procs) { pmix_show_help("help-prte-rmaps-seq.txt", "seq:not-enough-resources", true, app->num_procs, num_nodes); rc = PRTE_ERR_SILENT; goto error; } if (seq_list == &default_seq_list) { /* we don't want to start in the same place for every app, so take * the next item in the list - if this is the first time we are * using the default list, then start at its beginning */ if (NULL == save) { sq = (seq_node_t *) pmix_list_get_first(&default_seq_list); } else { sq = save; } } else { sq = (seq_node_t *) pmix_list_get_first(seq_list); } for (n = 0; n < app->num_procs; n++) { /* find this node on the global array - this is necessary so * that our mapping gets saved on that array as the objects * returned by the hostfile function are -not- on the array */ match = false; for (j = 0; j < prte_node_pool->size; j++) { node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, j); if (NULL == node) { continue; } if (prte_quickmatch(node, sq->hostname)) { match = true; break; } } if (!match) { /* wasn't found - that is an error */ pmix_show_help("help-prte-rmaps-seq.txt", "prte-rmaps-seq:resource-not-found", true, sq->hostname); rc = PRTE_ERR_SILENT; goto error; } /* check availability */ prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto error; } if (!prte_rmaps_base_check_avail(jdata, app, node, seq_list, NULL, options)) { continue; } /* map the proc */ proc = prte_rmaps_base_setup_proc(jdata, i, node, NULL, options); if (NULL == proc) { pmix_show_help("help-prte-rmaps-seq.txt", "proc-failed-to-map", true, sq->hostname, app->app); rc = PRTE_ERR_SILENT; goto error; } proc->name.rank = vpid; vpid++; proc->app_rank = apprank; apprank++; PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(proc); goto error; } rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_SUCCESS != rc && PRTE_ERR_TAKE_NEXT_OPTION != rc) { PMIX_RELEASE(proc); goto error; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:seq: assigned proc %s to node %s for app %s", PRTE_VPID_PRINT(proc->name.rank), sq->hostname, app->app); /* move to next node */ sq = (seq_node_t *) pmix_list_get_next(&sq->super); PMIX_RELEASE(proc); } /** track the total number of processes we mapped */ jdata->num_procs += app->num_procs; /* cleanup the node list if it came from this app_context */ if (seq_list != &default_seq_list) { PMIX_LIST_DESTRUCT(seq_list); } else { save = sq; } } /* compute local/app ranks */ rc = prte_rmaps_base_compute_vpids(jdata, options); return rc; error: PMIX_LIST_DESTRUCT(&default_seq_list); if (NULL != hosts) { free(hosts); } if (PRTE_ERR_SILENT != rc) { pmix_show_help("help-prte-rmaps-base.txt", "failed-map", true, PRTE_ERROR_NAME(rc), (NULL == app) ? "N/A" : app->app, (NULL == app) ? -1 : app->num_procs, prte_rmaps_base_print_mapping(options->map), prte_hwloc_base_print_binding(options->bind)); } return PRTE_ERR_SILENT; } static int process_file(char *path, pmix_list_t *list) { char *hstname = NULL; FILE *fp; seq_node_t *sq; char *sep, *eptr, *membind_opt; /* open the file */ fp = fopen(path, "r"); if (NULL == fp) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } while (NULL != (hstname = pmix_getline(fp))) { if (0 == strlen(hstname)) { free(hstname); /* blank line - ignore */ continue; } if ('#' == hstname[0]) { free(hstname); /* Comment line - ignore */ continue; } sq = PMIX_NEW(seq_node_t); if (NULL != (sep = strchr(hstname, ' '))) { *sep = '\0'; sep++; /* remove any trailing space */ eptr = sep + strlen(sep) - 1; while (eptr > sep && isspace(*eptr)) { eptr--; } *(eptr + 1) = 0; /* * If the submitted LSF job has memory binding related resource requirement, after * the cpu id list there are memory binding options. * * The following is the format of LSB_AFFINITY_HOSTFILE file: * * Host1 0,1,2,3 0 2 * Host1 4,5,6,7 1 2 * * Each line includes: host_name, cpu_id_list, NUMA_node_id_list, and memory_policy. * In this fix we will drop the last two sections (NUMA_node_id_list and memory_policy) * of each line and keep them in 'membind_opt' for future use. */ if (NULL != (membind_opt = strchr(sep, ' '))) { *membind_opt = '\0'; membind_opt++; } sq->cpuset = strdup(sep); } sq->hostname = hstname; pmix_list_append(list, &sq->super); } fclose(fp); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/seq/Makefile.am0000664000175000017500000000341715203542132021141 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-prte-rmaps-seq.txt sources = \ rmaps_seq.c \ rmaps_seq.h \ rmaps_seq_component.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_rmaps_seq_DSO component_noinst = component_install = prte_mca_rmaps_seq.la else component_noinst = libprtemca_rmaps_seq.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_rmaps_seq_la_SOURCES = $(sources) prte_mca_rmaps_seq_la_LDFLAGS = -module -avoid-version prte_mca_rmaps_seq_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_rmaps_seq_la_SOURCES =$(sources) libprtemca_rmaps_seq_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/rmaps/base/0000775000175000017500000000000015203542132017222 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/base/owner.txt0000664000175000017500000000024415203542132021115 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/rmaps/base/help-mapby.txt0000664000175000017500000001502215203542132022021 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRRTE's prun. # [map-by] Processes are mapped based on one of the following directives as applied at the job level: * "SLOT" assigns procs to each node up to the number of available slots on that node before moving to the next node in the allocation * "HWTHREAD" assigns a proc to each hardware thread on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "CORE" (default) assigns a proc to each core on a node in a round- robin manner up to the number of available slots on that node before moving to the next node in the allocation * "L1CACHE" assigns a proc to each L1 cache on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "L2CACHE" assigns a proc to each L2 cache on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "L3CACHE" assigns a proc to each L3 cache on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "NUMA" assigns a proc to each NUMA region on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "PACKAGE" assigns a proc to each package on a node in a round-robin manner up to the number of available slots on that node before moving to the next node in the allocation * "NODE" assigns processes in a round-robin fashion to all nodes in the allocation, with the number assigned to each node capped by the number of available slots on that node * "SEQ" (often accompanied by the file= qualifier) assigns one process to each node specified in the file. The sequential file is to contain an entry for each desired process, one per line of the file. * "PPR:N":resource maps N procs to each instance of the specified resource type in the allocation * "RANKFILE" (often accompanied by the file= qualifier) assigns one process to the node/resource specified in each entry of the file, one per line of the file. * "PE-LIST=a,b" assigns procs to each node in the allocation based on the ORDERED qualifier. The list is comprised of comma-delimited ranges of CPUs to use for this job. If the ORDERED qualifier is not provided, then each node will be assigned procs up to the number of available slots, capped by the total number of specified processes and availability of the specified CPUs (i.e. the number of procs cannot exceed the number of specified CPUs), with each proc bound to all the specified CPUs. If ORDERED is given, then one proc will be assigned to each of the specified CPUs, if available, capped by the number of slots on each node and the total number of specified processes. Providing the OVERLOAD qualifier to the "bind-to" option removes the check on availability of the CPU in both cases. Any directive can include qualifiers by adding a colon (":") and any combination of one or more of the following (delimited by colons) to the "--map-by" option (except where noted): * "PE=n" bind n CPUs to each process (can not be used in combination with rankfile or pe-list directives) * "SPAN" load balance the processes across the allocation by treating the allocation as a single "super-node" (can not be used in combination with "slot", "node", "seq", "ppr", "rankfile", or "pe- list" directives) * "OVERSUBSCRIBE" allow more processes on a node than processing elements * "NOOVERSUBSCRIBE" means "!OVERSUBSCRIBE" * "NOLOCAL" do not launch processes on the same node as "prun" * "HWTCPUS" use hardware threads as CPU slots * "CORECPUS" use cores as CPU slots (default) * "INHERIT" indicates that a child job (i.e., one spawned from within an application) shall inherit the placement policies of the parent job that spawned it. * "NOINHERIT" means "`!INHERIT" * "FILE=" (path to file containing sequential or rankfile entries). * "ORDERED" only applies to the "PE-LIST" option to indicate that procs are to be bound to each of the specified CPUs in the order in which they are assigned (i.e., the first proc on a node shall be bound to the first CPU in the list, the second proc shall be bound to the second CPU, etc.) Note: Directives and qualifiers are case-insensitive and can be shortened to the minimum number of characters to uniquely identify them. Thus, "L1CACHE" can be given as "l1cache" or simply as "L1". The type of CPU (core vs hwthread) used in the mapping algorithm is determined as follows: * by user directive on the command line via the HWTCPUS qualifier to the "--map-by" directive * by setting the "rmaps_default_mapping_policy" MCA parameter to include the "HWTCPUS" qualifier. This parameter sets the default value for a PRRTE DVM — qualifiers are carried across to DVM jobs started via "prun" unless overridden by the user's command line * defaults to CORE in topologies where core CPUs are defined, and to hwthreads otherwise. If your application uses threads, then you probably want to ensure that you are either not bound at all (by specifying "--bind-to none"), or bound to multiple cores using an appropriate binding level or specific number of processing elements per application process via the "PE=#" qualifier to the "--map-by" command line directive. A more detailed description of the mapping, ranking, and binding procedure can be obtained via the "--help placement" option. prrte-4.1.1~rc2/src/mca/rmaps/base/help-prte-rmaps-base.txt0000664000175000017500000003407115203542132023720 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRRTE's prun. # [prte-rmaps-base:alloc-error] There are not enough slots available in the system to satisfy the %d slots that were requested by the application: %s Either request fewer procs for your application, or make more slots available for use. A "slot" is the PRRTE term for an allocatable unit where we can launch a process. The number of slots available are defined by the environment in which PRRTE processes are run: 1. Hostfile, via "slots=N" clauses (N defaults to number of processor cores if not provided) 2. The --host command line parameter, via a ":N" suffix on the hostname (N defaults to 1 if not provided) 3. Resource manager (e.g., SLURM, PBS/Torque, LSF, etc.) 4. If none of a hostfile, the --host command line parameter, or an RM is present, PRRTE defaults to the number of processor cores In all the above cases, if you want PRRTE to default to the number of hardware threads instead of the number of processor cores, use the --use-hwthread-cpus option. Alternatively, you can use the --map-by :OVERSUBSCRIBE option to ignore the number of available slots when deciding the number of processes to launch. # [prte-rmaps-base:no-mapped-node] There are no allocated resources for the application: %s that match the requested mapping: %s: %s Verify that you have mapped the allocated resources properly for the indicated specification. # [prte-rmaps-base:no-available-resources] No nodes are available for this job, either due to a failure to allocate nodes to the job, or allocated nodes being marked as unavailable (e.g., down, rebooting, or a process attempting to be relocated to another node when none are available). [prte-rmaps-base:all-available-resources-used] All nodes which are allocated for this job are already filled. # [out-of-vpids] The system has exhausted its available ranks - the application is attempting to spawn too many daemons and will be aborted. This may be resolved by increasing the number of available ranks by re-configuring with the --enable-jumbo-apps option, and then re-building the application. # [failed-map] Your job failed to map. Either no mapper was available, or none of the available mappers was able to perform the requested mapping operation. Mapper result: %s Application: %s #procs to be mapped: %d Mapping policy: %s Binding policy: %s # [span-packages-multiple] Your job failed to map because either a package with sufficient available cpus could not be found on the node, or the resulting process placement would cause the process to be bound to CPUs in more than one package on the node: Mapping policy: %s Binding policy: %s CPUs/rank: %d Node: %s If we couldn't find a package on the node that had sufficient available cpus, then you may need to alter your mapping request. For example, your ppr request may include more procs than the node can support. If we failed to map due to cross-package placement, then you may need to alter your policy. A configuration that causes a single process to operate across multiple packages almost always results in a loss of performance that can significantly impact applications. Either alter the mapping, binding, and/or cpus/rank policies so that each process can fit into a single package, or consider using an alternative mapper that can handle this configuration (e.g., the rankfile mapper). # [span-packages-cpuset] Your job failed to map because the resulting process placement would cause the process to be bound to CPUs in more than one package: Mapping policy: %s Binding policy: %s PE-LIST: %s This configuration almost always results in a loss of performance that can significantly impact applications. Either alter the mapping, binding, and/or PE-LIST policies so that each process can fit into a single package, or consider using an alternative mapper that can handle this configuration (e.g., the rankfile mapper). # [unrecognized-policy] The specified %s directive is not recognized: Directive: %s Please check for a typo or ensure that the directive is a supported one. # [rmaps:binding-target-not-found] A request was made to bind to %s, but an appropriate target could not be found on node %s. # [allocation-overload] A request was made to bind that would require binding processes to more cpus than are available in your allocation: Application: %s #processes: %d Mapping policy: %s Binding policy: %s You can override this protection by adding the "overload-allowed" option to your binding directive. # [rmaps:no-topology] A mapping directive was given that requires knowledge of a remote node's topology. However, no topology info is available for the following node: Node: %s The job cannot be executed under this condition. Please either remove the directive or investigate the lack of topology info. # [rmaps:no-available-cpus] While computing bindings, we found no available cpus on the following node: Node: %s Please check your allocation. # [rmaps:cpubind-not-supported] A request was made to bind a process, but at least one node does NOT support binding processes to cpus. Node: %s PRRTE uses the "hwloc" library to perform process and memory binding. This error message means that hwloc has indicated that processor binding support is not available on this machine. On OS X, processor and memory binding is not available at all (i.e., the OS does not expose this functionality). On Linux, lack of the functionality can mean that you are on a platform where processor and memory affinity is not supported in Linux itself, or that hwloc was built without NUMA and/or processor affinity support. When building hwloc (which, depending on your PRRTE installation, may be embedded in PRRTE itself), it is important to have the libnuma header and library files available. Different linux distributions package these files under different names; look for packages with the word "numa" in them. You may also need a developer version of the package (e.g., with "dev" or "devel" in the name) to obtain the relevant header files. If you are getting this message on a non-OS X, non-Linux platform, then hwloc does not support processor / memory affinity on this platform. If the OS/platform does actually support processor / memory affinity, then you should contact the hwloc maintainers: https://github.com/open-mpi/hwloc. # [rmaps:membind-not-supported] WARNING: a request was made to bind a process. While the system supports binding the process itself, at least one node does NOT support binding memory to the process location. Node: %s PRRTE uses the "hwloc" library to perform process and memory binding. This error message means that hwloc has indicated that processor binding support is not available on this machine. On OS X, processor and memory binding is not available at all (i.e., the OS does not expose this functionality). On Linux, lack of the functionality can mean that you are on a platform where processor and memory affinity is not supported in Linux itself, or that hwloc was built without NUMA and/or processor affinity support. When building hwloc (which, depending on your PRRTE installation, may be embedded in PRRTE itself), it is important to have the libnuma header and library files available. Different linux distributions package these files under different names; look for packages with the word "numa" in them. You may also need a developer version of the package (e.g., with "dev" or "devel" in the name) to obtain the relevant header files. If you are getting this message on a non-OS X, non-Linux platform, then hwloc does not support processor / memory affinity on this platform. If the OS/platform does actually support processor / memory affinity, then you should contact the hwloc maintainers: https://github.com/open-mpi/hwloc. This is a warning only; your job will continue, though performance may be degraded. # [rmaps:membind-not-supported-fatal] A request was made to bind a process. While the system supports binding the process itself, at least one node does NOT support binding memory to the process location. Node: %s PRRTE uses the "hwloc" library to perform process and memory binding. This error message means that hwloc has indicated that processor binding support is not available on this machine. On OS X, processor and memory binding is not available at all (i.e., the OS does not expose this functionality). On Linux, lack of the functionality can mean that you are on a platform where processor and memory affinity is not supported in Linux itself, or that hwloc was built without NUMA and/or processor affinity support. When building hwloc (which, depending on your PRRTE installation, may be embedded in PRRTE itself), it is important to have the libnuma header and library files available. Different linux distributions package these files under different names; look for packages with the word "numa" in them. You may also need a developer version of the package (e.g., with "dev" or "devel" in the name) to obtain the relevant header files. If you are getting this message on a non-OS X, non-Linux platform, then hwloc does not support processor / memory affinity on this platform. If the OS/platform does actually support processor / memory affinity, then you should contact the hwloc maintainers: https://github.com/open-mpi/hwloc. The provided memory binding policy requires that PRRTE abort the job at this time. # [mapping-too-low] A request for multiple cpus-per-proc was given, but a directive was also given to map to an object level that has less cpus than requested ones: #cpus-per-proc: %d number of cpus: %d map-by: %s Please specify a mapping level that has more cpus, or else let us define a default mapping that will allow multiple cpus-per-proc. # [unrecognized-modifier] The mapping request contains an unrecognized qualifier: Request: %s Please check your request and try again. Remember, multiple qualifiers must be separated by a colon (':'). # [invalid-pattern] The mapping request contains a pattern that doesn't match the required syntax of #:object Pattern: %s Please check your request and try again. # [cannot-launch] Although we were able to map your job, we are unable to launch it at this time due to required resources being busy. Please try again later. # [unsupported-default-modifier] A %s modifier was provided that is not supported as a default value: Modifier: %s You can provide this modifier on a per-job basis, but it cannot be the default setting. # [unsupported-default-policy] A %s policy was provided that is not supported as a default value: Policy: %s You can provide this policy on a per-job basis, but it cannot be the default setting. # [missing-value] A %s was given that requires a value, but no value for it was given or the value is missing the required '=' sign: Policy: %s Given: %s Please provide a value for this policy, or remove it. # [invalid-value] A %s was given that requires a value, but the provided value is invalid: Policy: %s Given: %s Please provide a valid value for this policy, or remove it. # [rankfile-no-filename] The request to map processes using a rankfile could not be completed because the filename of the rankfile was not specified. Please specify the name of the rankfile. # [unrecognized-qualifier] The %s directive contains an unrecognized qualifier: Qualifier: %s Valid qualifiers: %s Please check for a typo or ensure that the qualifier is a supported one. # [unrecognized-directive] The specified %s directive is not recognized: Directive: %s Valid directives: %s Please check for a typo or ensure that the directive is a supported one. # [missing-qualifier] The %s option contains a directive that is missing a value: Directive: %s Valid directive: "%s=" Please check for a typo or ensure that the value is provided. # [missing-personality] PRRTE has hit an internal problem that should never happen - it is attempting to map a job that lacks an assigned personality. Job: %s Please report this to the PRRTE developers (https://github.com/openpmix/prrte/issues) # [unsupported-combination] A %s policy was provided that is not supported in combination with the "PE=N" option: Policy: %s When specifying the number of CPUs to use for each process in a job, the processes must be bound at the CPU level - either HWThread if "HWTCPUS" was specified, or CORE. Please remove the bind policy specification and try again. # [redefining-policy] Conflicting directives for binding policy are causing the policy to be redefined: New policy: %s Prior policy: %s Please check that only one policy is defined. # [multiply-defined] A %s was given that defines a value multiple times: Policy: %s Given: %s Given: %s Please provide only one value for this policy, or remove it. # [not-enough-cpus] A request was made to map a specified number of processes to each object of a given type, and to bind each process to a given number of CPUs with that object. Unfortunately, at least one such object lacks enough CPUs to meet that combination of requirements: #procs/obj: %d Object type: %s cpus/proc: %d Please adjust the request and try again. prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_print_fns.c0000664000175000017500000001422115203542132023564 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2018-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/threads/pmix_tsd.h" #include "src/util/pmix_if.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/runtime/prte_globals.h" #include "src/util/dash_host/dash_host.h" #include "src/util/hostfile/hostfile.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "types.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" #define PRTE_RMAPS_PRINT_MAX_SIZE 50 #define PRTE_RMAPS_PRINT_NUM_BUFS 16 static bool fns_init = false; static pmix_tsd_key_t print_tsd_key; static char *prte_rmaps_print_null = "NULL"; typedef struct { char *buffers[PRTE_RMAPS_PRINT_NUM_BUFS]; int cntr; } prte_rmaps_print_buffers_t; static void buffer_cleanup(void *value) { int i; prte_rmaps_print_buffers_t *ptr; if (NULL != value) { ptr = (prte_rmaps_print_buffers_t *) value; for (i = 0; i < PRTE_RMAPS_PRINT_NUM_BUFS; i++) { free(ptr->buffers[i]); } } } static prte_rmaps_print_buffers_t *get_print_buffer(void) { prte_rmaps_print_buffers_t *ptr; int ret, i; if (!fns_init) { /* setup the print_args function */ if (PRTE_SUCCESS != (ret = pmix_tsd_key_create(&print_tsd_key, buffer_cleanup))) { PRTE_ERROR_LOG(ret); return NULL; } fns_init = true; } ret = pmix_tsd_getspecific(print_tsd_key, (void **) &ptr); if (PRTE_SUCCESS != ret) return NULL; if (NULL == ptr) { ptr = (prte_rmaps_print_buffers_t *) malloc(sizeof(prte_rmaps_print_buffers_t)); for (i = 0; i < PRTE_RMAPS_PRINT_NUM_BUFS; i++) { ptr->buffers[i] = (char *) malloc((PRTE_RMAPS_PRINT_MAX_SIZE + 1) * sizeof(char)); } ptr->cntr = 0; ret = pmix_tsd_setspecific(print_tsd_key, (void *) ptr); } return (prte_rmaps_print_buffers_t *) ptr; } char *prte_rmaps_base_print_mapping(prte_mapping_policy_t mapping) { char *ret, *map, *mymap, *tmp; char **qls = NULL; prte_rmaps_print_buffers_t *ptr; ptr = get_print_buffer(); if (NULL == ptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return prte_rmaps_print_null; } /* cycle around the ring */ if (PRTE_RMAPS_PRINT_NUM_BUFS == ptr->cntr) { ptr->cntr = 0; } switch (PRTE_GET_MAPPING_POLICY(mapping)) { case PRTE_MAPPING_BYNODE: map = "BYNODE"; break; case PRTE_MAPPING_BYPACKAGE: map = "BYPACKAGE"; break; case PRTE_MAPPING_BYNUMA: map = "BYNUMA"; break; case PRTE_MAPPING_BYL3CACHE: map = "BYL3CACHE"; break; case PRTE_MAPPING_BYL2CACHE: map = "BYL2CACHE"; break; case PRTE_MAPPING_BYL1CACHE: map = "BYL1CACHE"; break; case PRTE_MAPPING_BYCORE: map = "BYCORE"; break; case PRTE_MAPPING_BYHWTHREAD: map = "BYHWTHREAD"; break; case PRTE_MAPPING_BYSLOT: map = "BYSLOT"; break; case PRTE_MAPPING_SEQ: map = "SEQUENTIAL"; break; case PRTE_MAPPING_COLOCATE: map = "COLOCATE"; break; case PRTE_MAPPING_BYUSER: map = "BYUSER"; break; case PRTE_MAPPING_BYDIST: map = "MINDIST"; break; case PRTE_MAPPING_PELIST: map = "PE-LIST"; break; case PRTE_MAPPING_PPR: map = "PPR"; break; default: map = "UNKNOWN"; } if (PRTE_MAPPING_NO_USE_LOCAL & PRTE_GET_MAPPING_DIRECTIVE(mapping)) { PMIx_Argv_append_nosize(&qls, "NO_USE_LOCAL"); } if (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(mapping)) { PMIx_Argv_append_nosize(&qls, "NOOVERSUBSCRIBE"); } else if (PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(mapping)) { PMIx_Argv_append_nosize(&qls, "OVERSUBSCRIBE"); } if (PRTE_MAPPING_SPAN & PRTE_GET_MAPPING_DIRECTIVE(mapping)) { PMIx_Argv_append_nosize(&qls, "SPAN"); } if (PRTE_MAPPING_ORDERED & PRTE_GET_MAPPING_DIRECTIVE(mapping)) { PMIx_Argv_append_nosize(&qls, "ORDERED"); } if (NULL != qls) { tmp = PMIx_Argv_join(qls, ':'); PMIx_Argv_free(qls); pmix_asprintf(&mymap, "%s:%s", map, tmp); free(tmp); } else { mymap = strdup(map); } snprintf(ptr->buffers[ptr->cntr], PRTE_RMAPS_PRINT_MAX_SIZE, "%s", mymap); free(mymap); ret = ptr->buffers[ptr->cntr]; ptr->cntr++; return ret; } char *prte_rmaps_base_print_ranking(prte_ranking_policy_t ranking) { char *ret; switch (PRTE_GET_RANKING_POLICY(ranking)) { case PRTE_RANK_BY_NODE: ret = "NODE"; break; case PRTE_RANK_BY_SLOT: ret = "SLOT"; break; case PRTE_RANK_BY_FILL: ret = "FILL"; break; case PRTE_RANK_BY_SPAN: ret = "SPAN"; break; case PRTE_RANKING_BYUSER: ret = "BYUSER"; break; default: ret = "UNKNOWN"; break; } return ret; } prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_map_job.c0000664000175000017500000017276715203542132023215 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2019 UT-Battelle, LLC. All rights reserved. * * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2022 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_output.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/iof/base/base.h" #include "src/mca/ras/base/base.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_show_help.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" static int map_colocate(prte_job_t *jdata, bool daemons, bool pernode, pmix_data_array_t *darray, uint16_t procs_per_target, prte_rmaps_options_t *options); static void inherit_env_directives(prte_job_t *jdata, prte_job_t *parent, pmix_proc_t *proxy); void prte_rmaps_base_map_job(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_node_t *node; pmix_proc_t *pptr; int rc = PRTE_SUCCESS; int n; bool did_map, pernode = false; prte_rmaps_base_selected_module_t *mod; prte_job_t *parent = NULL; prte_app_context_t *app; bool inherit = false; pmix_proc_t *nptr = NULL, *target_proc; char *tmp, **ck, **env; uint16_t u16 = 0, procs_per_target = 0; uint16_t *u16ptr = &u16; bool colocate_daemons = false; bool colocate = false; prte_schizo_base_module_t *schizo; prte_rmaps_options_t options; pmix_data_array_t *darray = NULL; pmix_list_t nodes; int slots, len; bool flag, *fptr; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); // init options memset(&options, 0, sizeof(prte_rmaps_options_t)); options.stream = prte_rmaps_base_framework.framework_output; options.verbosity = 5; // usual value for base-level functions // set and check convenience vars jdata = caddy->jdata; schizo = (prte_schizo_base_module_t*)jdata->schizo; if (NULL == schizo) { pmix_show_help("help-prte-rmaps-base.txt", "missing-personality", true, PRTE_JOBID_PRINT(jdata->nspace)); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } if (NULL == jdata->map) { jdata->map = PMIX_NEW(prte_job_map_t); } jdata->state = PRTE_JOB_STATE_MAP; fptr = &flag; /* check and set some general options */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { options.donotlaunch = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_MAP, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL)) { options.dobind = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_BINDING_LIMIT, (void**) &u16ptr, PMIX_UINT16)) { options.limit = u16; // reset any prior counters prte_hwloc_base_reset_counters(); } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: mapping job %s", PRTE_JOBID_PRINT(jdata->nspace)); /* * Check for Colaunch */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DEBUG_DAEMONS_PER_NODE, (void **) &u16ptr, PMIX_UINT16)) { procs_per_target = u16; if (procs_per_target == 0) { pmix_output(0, "Error: PRTE_JOB_DEBUG_DAEMONS_PER_NODE value %u == 0\n", procs_per_target); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } pernode = true; colocate_daemons = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DEBUG_DAEMONS_PER_PROC, (void **) &u16ptr, PMIX_UINT16)) { if (procs_per_target > 0) { pmix_output(0, "Error: Both PRTE_JOB_DEBUG_DAEMONS_PER_PROC and " "PRTE_JOB_DEBUG_DAEMONS_PER_NODE provided."); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } procs_per_target = u16; if (procs_per_target == 0) { pmix_output(0, "Error: PRTE_JOB_DEBUG_DAEMONS_PER_PROC value %u == 0\n", procs_per_target); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } pernode = false; colocate_daemons = true; } if (colocate_daemons) { if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_DEBUG_TARGET, (void **) &target_proc, PMIX_PROC)) { pmix_output(0, "Error: PRTE_JOB_DEBUG_DAEMONS_PER_PROC/NODE provided without a Debug Target\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* store the target as a pmix_data_array_t */ PMIX_DATA_ARRAY_CREATE(darray, 1, PMIX_PROC); pptr = (pmix_proc_t*)darray->array; PMIX_XFER_PROCID(&pptr[0], target_proc); } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_COLOCATE_PROCS, (void**)&darray, PMIX_DATA_ARRAY)) { if (colocate_daemons) { pmix_output(0, "Error: Both colocate daemons and colocate procs were provided\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } if (NULL == darray) { pmix_output(0, "Error: Colocate failed to provide procs\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } colocate = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_COLOCATE_NPERNODE, (void **) &u16ptr, PMIX_UINT16)) { procs_per_target = u16; if (procs_per_target == 0) { pmix_output(0, "Error: PRTE_JOB_COLOCATE_NUM_PROC WITH ZERO PROCS/TARGET\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } pernode = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_COLOCATE_NPERPROC, (void **) &u16ptr, PMIX_UINT16)) { if (procs_per_target > 0) { pmix_output(0, "Error: Both PRTE_JOB_COLOCATE_NUM_PROC and " "PRTE_JOB_COLOCATE_NUM_NODE provided."); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } procs_per_target = u16; if (procs_per_target == 0) { pmix_output(0, "Error: PRTE_JOB_COLOCATE_NUM_PROC WITH ZERO PROCS/TARGET\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } pernode = false; } if (colocate || colocate_daemons) { PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_COLOCATE); goto ranking; } /* if this is a dynamic job launch and they didn't explicitly * request inheritance, then don't inherit the launch directives */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC)) { if (NULL == nptr) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* if the launch proxy is me, then this is the initial launch from * a proxy scenario, so we don't really have a parent */ if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, nptr->nspace)) { parent = NULL; /* we do allow inheritance of the defaults */ inherit = true; } else if (NULL != (parent = prte_get_job_data_object(nptr->nspace))) { if (PRTE_FLAG_TEST(parent, PRTE_JOB_FLAG_TOOL)) { // we don't inherit anything from tools as they were not // mapped by us inherit = false; parent = NULL; } else if (prte_get_attribute(&parent->attributes, PRTE_JOB_INHERIT, NULL, PMIX_BOOL)) { inherit = true; // if they didn't specifically direct it not inherit, then pass this on to the child if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_NOINHERIT, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_ATTR_GLOBAL, PRTE_JOB_INHERIT, NULL, PMIX_BOOL); } } else if (prte_get_attribute(&parent->attributes, PRTE_JOB_NOINHERIT, NULL, PMIX_BOOL)) { inherit = false; parent = NULL; } else { inherit = prte_rmaps_base.inherit; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: dynamic job %s %s inherit launch directives - parent %s", PRTE_JOBID_PRINT(jdata->nspace), inherit ? "will" : "will not", (NULL == parent) ? "N/A" : PRTE_JOBID_PRINT((parent->nspace))); } else { inherit = true; } } else { /* initial launch always takes on default MCA params for non-specified policies */ inherit = true; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: setting mapping policies for job %s inherit %s hwtcpus %s", PRTE_JOBID_PRINT(jdata->nspace), inherit ? "TRUE" : "FALSE", options.use_hwthreads ? "TRUE" : "FALSE"); /* set the default mapping policy IFF it wasn't provided */ if (!PRTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) { if (inherit) { if (NULL != parent) { // copy across the mapping policy jdata->map->mapping = parent->map->mapping; /* if not already assigned, inherit the parent's ppr */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PPR, NULL, PMIX_STRING)) { /* get the parent job's ppr, if it had one */ if (prte_get_attribute(&parent->attributes, PRTE_JOB_PPR, (void **) &tmp, PMIX_STRING)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_PPR, PRTE_ATTR_GLOBAL, tmp, PMIX_STRING); free(tmp); } else if (NULL != prte_rmaps_base.ppr) { prte_set_attribute(&jdata->attributes, PRTE_JOB_PPR, PRTE_ATTR_GLOBAL, prte_rmaps_base.ppr, PMIX_STRING); } } /* if not already assigned, inherit the parent's pes/proc */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16)) { /* get the parent job's pes/proc, if it had one */ if (prte_get_attribute(&parent->attributes, PRTE_JOB_PES_PER_PROC, (void **) &u16ptr, PMIX_UINT16)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, PRTE_ATTR_GLOBAL, u16ptr, PMIX_UINT16); } else if (0 < prte_rmaps_base.default_pes) { u16 = prte_rmaps_base.default_pes; prte_set_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, PRTE_ATTR_GLOBAL, u16ptr, PMIX_UINT16); } } /* if not already assigned, inherit the parent's cpu designation */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL) && !prte_get_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, NULL, PMIX_BOOL)) { /* get the parent job's designation, if it had one */ if (prte_get_attribute(&parent->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else if (prte_get_attribute(&parent->attributes, PRTE_JOB_CORE_CPUS, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else { /* default */ if (prte_rmaps_base.hwthread_cpus) { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } } /* if not already assigned, inherit the parent's GPU support directive */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_GPU_SUPPORT, NULL, PMIX_BOOL)) { if (prte_get_attribute(&parent->attributes, PRTE_JOB_GPU_SUPPORT, (void **) &fptr, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_GPU_SUPPORT, PRTE_ATTR_GLOBAL, fptr, PMIX_BOOL); } } /* if not already assigned, inherit the parent's output directives */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_TAG_OUTPUT, NULL, PMIX_BOOL)) { if (prte_get_attribute(&parent->attributes, PRTE_JOB_TAG_OUTPUT, (void **) &fptr, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_TAG_OUTPUT, PRTE_ATTR_GLOBAL, fptr, PMIX_BOOL); } } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMESTAMP_OUTPUT, NULL, PMIX_BOOL)) { if (prte_get_attribute(&parent->attributes, PRTE_JOB_TIMESTAMP_OUTPUT, (void **) &fptr, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_TIMESTAMP_OUTPUT, PRTE_ATTR_GLOBAL, fptr, PMIX_BOOL); } } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_MERGE_STDERR_STDOUT, NULL, PMIX_BOOL)) { if (prte_get_attribute(&parent->attributes, PRTE_JOB_MERGE_STDERR_STDOUT, (void **) &fptr, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_MERGE_STDERR_STDOUT, PRTE_ATTR_GLOBAL, fptr, PMIX_BOOL); } } // copy over any env directives, but do not overwrite anything already specified inherit_env_directives(jdata, parent, nptr); } else { // bring over the MCA param defaults, where set and not already specified for this job if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PPR, NULL, PMIX_STRING) && NULL != prte_rmaps_base.ppr) { prte_set_attribute(&jdata->attributes, PRTE_JOB_PPR, PRTE_ATTR_GLOBAL, prte_rmaps_base.ppr, PMIX_STRING); } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16) && 0 < prte_rmaps_base.default_pes) { u16 = prte_rmaps_base.default_pes; prte_set_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, PRTE_ATTR_GLOBAL, u16ptr, PMIX_UINT16); options.cpus_per_rank = u16; } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL) && !prte_get_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, NULL, PMIX_BOOL)) { if (prte_rmaps_base.hwthread_cpus) { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL) && !prte_get_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, NULL, PMIX_BOOL)) { /* inherit the base defaults */ if (prte_rmaps_base.hwthread_cpus) { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, NULL, PMIX_STRING) && NULL != prte_rmaps_base.file) { prte_set_attribute(&jdata->attributes, PRTE_JOB_FILE, PRTE_ATTR_GLOBAL, prte_rmaps_base.file, PMIX_STRING); } if (PRTE_MAPPING_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping)) { jdata->map->mapping = prte_rmaps_base.mapping; } else { // let the job's personality set the default mapping behavior if (NULL != schizo->set_default_mapping) { rc = schizo->set_default_mapping(jdata, &options); } else { rc = prte_rmaps_base_set_default_mapping(jdata, &options); } if (PRTE_SUCCESS != rc) { // the error message should have been printed jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } } } else { // let the job's personality set the default mapping behavior if (NULL != schizo->set_default_mapping) { rc = schizo->set_default_mapping(jdata, &options); } else { rc = prte_rmaps_base_set_default_mapping(jdata, &options); } if (PRTE_SUCCESS != rc) { // the error message should have been printed jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } } if (NULL != nptr) { PMIX_PROC_RELEASE(nptr); } /* we always inherit a parent's oversubscribe flag unless the job assigned it */ if (NULL != parent && !(PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) { if (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(parent->map->mapping)) { PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); } else { PRTE_UNSET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_SUBSCRIBE_GIVEN); } } // forward the environment if requested to do so if (prte_get_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT, (void **) &fptr, PMIX_BOOL)) { if (flag) { // forward the environment for (n = 0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } env = pmix_environ_merge(prte_launch_environ, app->env); PMIx_Argv_free(app->env); app->env = env; } } } else if (NULL != parent) { /* we always inherit a parent's fwd environment directive unless the job assigned it */ if (prte_get_attribute(&parent->attributes, PRTE_JOB_FWD_ENVIRONMENT, (void **) &fptr, PMIX_BOOL)) { if (flag) { // update the child's flag so any subsequent children can inherit it prte_set_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); // forward the environment for (n = 0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } env = pmix_environ_merge(prte_launch_environ, app->env); PMIx_Argv_free(app->env); app->env = env; } } } } /* set some convenience params */ prte_get_attribute(&jdata->attributes, PRTE_JOB_CPUSET, (void**)&options.cpuset, PMIX_STRING); if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, (void **) &u16ptr, PMIX_UINT16)) { options.cpus_per_rank = u16; } else { options.cpus_per_rank = 1; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { options.use_hwthreads = true; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PROCESSORS, (void*)&tmp, PMIX_STRING)) { prte_ras_base_display_cpus(jdata, tmp); free(tmp); } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PPR, (void **) &tmp, PMIX_STRING)) { ck = PMIx_Argv_split(tmp, ':'); if (2 != PMIx_Argv_count(ck)) { /* must provide a specification */ pmix_show_help("help-prte-rmaps-ppr.txt", "invalid-ppr", true, tmp); PMIx_Argv_free(ck); free(tmp); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* compute the #procs per resource */ options.pprn = strtoul(ck[0], NULL, 10); len = strlen(ck[1]); if (0 == strncasecmp(ck[1], "node", len)) { options.maptype = HWLOC_OBJ_MACHINE; options.mapdepth = PRTE_BIND_TO_NONE; } else if (0 == strncasecmp(ck[1], "hwthread", len) || 0 == strncasecmp(ck[1], "thread", len)) { options.maptype = HWLOC_OBJ_PU; options.mapdepth = PRTE_BIND_TO_HWTHREAD; } else if (0 == strncasecmp(ck[1], "core", len)) { options.maptype = HWLOC_OBJ_CORE; options.mapdepth = PRTE_BIND_TO_CORE; } else if (0 == strncasecmp(ck[1], "package", len) || 0 == strncasecmp(ck[1], "skt", len) || 0 == strncasecmp(ck[1], "socket", len)) { options.maptype = HWLOC_OBJ_PACKAGE; options.mapdepth = PRTE_BIND_TO_PACKAGE; } else if (0 == strncasecmp(ck[1], "numa", len) || 0 == strncasecmp(ck[1], "nm", len)) { options.maptype = HWLOC_OBJ_NUMANODE; options.mapdepth = PRTE_BIND_TO_NUMA; } else if (0 == strncasecmp(ck[1], "l1cache", len)) { options.maptype = HWLOC_OBJ_L1CACHE; options.mapdepth = PRTE_BIND_TO_L1CACHE; } else if (0 == strncasecmp(ck[1], "l2cache", len)) { options.maptype = HWLOC_OBJ_L2CACHE; options.mapdepth = PRTE_BIND_TO_L2CACHE; } else if (0 == strncasecmp(ck[1], "l3cache", len)) { options.maptype = HWLOC_OBJ_L3CACHE; options.mapdepth = PRTE_BIND_TO_L3CACHE; } else { /* unknown spec */ pmix_show_help("help-prte-rmaps-ppr.txt", "unrecognized-ppr-option", true, ck[1], tmp); free(tmp); PMIx_Argv_free(ck); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } free(tmp); PMIx_Argv_free(ck); } /* add up all the expected procs */ for (n = 0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app ) { continue; } if (0 < app->num_procs) { options.nprocs += app->num_procs; continue; } PRTE_FLAG_SET(app, PRTE_APP_FLAG_COMPUTED); if (PRTE_MAPPING_SEQ == PRTE_GET_MAPPING_POLICY(jdata->map->mapping) || PRTE_MAPPING_BYUSER == PRTE_GET_MAPPING_POLICY(jdata->map->mapping) || PRTE_MAPPING_PPR == PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) { // these mappers compute their #procs as they go continue; } if (NULL != options.cpuset) { ck = PMIx_Argv_split(options.cpuset, ','); app->num_procs = PMIx_Argv_count(ck); PMIx_Argv_free(ck); } else { /* * get the target nodes for this app - the base function * will take any host or hostfile directive into account */ PMIX_CONSTRUCT(&nodes, pmix_list_t); rc = prte_rmaps_base_get_target_nodes(&nodes, &slots, jdata, app, jdata->map->mapping, true, true, false); if (PRTE_SUCCESS != rc) { PMIX_LIST_DESTRUCT(&nodes); jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } if (1 < options.cpus_per_rank) { // compute the number of cpus on each node len = 0; PMIX_LIST_FOREACH (node, &nodes, prte_node_t) { if (options.use_hwthreads) { len += prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PU) / options.cpus_per_rank; } else { len += prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_CORE) / options.cpus_per_rank; } } app->num_procs = len; // ensure we always wind up with at least one proc if (0 == app->num_procs) { app->num_procs = 1; } else if (slots < app->num_procs) { app->num_procs = slots; } } else { app->num_procs = slots; } PMIX_LIST_DESTRUCT(&nodes); } options.nprocs += app->num_procs; } /* check for oversubscribe directives */ if (!(PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) { if (!(PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping))) { PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); } else if (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping)) { PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); } else { PRTE_UNSET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_SUBSCRIBE_GIVEN); } } if (!(PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) { options.oversubscribe = true; } /* check for no-use-local directive */ if (prte_ras_base.launch_orted_on_hn) { /* must override any setting */ PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_USE_LOCAL); } else if (!(PRTE_MAPPING_LOCAL_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) { if (inherit && (PRTE_MAPPING_NO_USE_LOCAL & PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping))) { PRTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, PRTE_MAPPING_NO_USE_LOCAL); } } ranking: options.map = PRTE_GET_MAPPING_POLICY(jdata->map->mapping); if (PRTE_MAPPING_SPAN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) { options.mapspan = true; } if (PRTE_MAPPING_ORDERED & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) { options.ordered = true; } switch (options.map) { case PRTE_MAPPING_BYNODE: case PRTE_MAPPING_BYSLOT: case PRTE_MAPPING_BYDIST: case PRTE_MAPPING_PELIST: case PRTE_MAPPING_COLOCATE: options.mapdepth = PRTE_BIND_TO_NONE; options.maptype = HWLOC_OBJ_MACHINE; break; case PRTE_MAPPING_BYUSER: case PRTE_MAPPING_SEQ: options.mapdepth = PRTE_BIND_TO_NONE; options.userranked = true; options.maptype = HWLOC_OBJ_MACHINE; break; case PRTE_MAPPING_BYNUMA: options.mapdepth = PRTE_BIND_TO_NUMA; options.maptype = HWLOC_OBJ_NUMANODE; break; case PRTE_MAPPING_BYPACKAGE: options.mapdepth = PRTE_BIND_TO_PACKAGE; options.maptype = HWLOC_OBJ_PACKAGE; break; case PRTE_MAPPING_BYL3CACHE: options.mapdepth = PRTE_BIND_TO_L3CACHE; options.maptype = HWLOC_OBJ_L3CACHE; break; case PRTE_MAPPING_BYL2CACHE: options.mapdepth = PRTE_BIND_TO_L2CACHE; options.maptype = HWLOC_OBJ_L2CACHE; break; case PRTE_MAPPING_BYL1CACHE: options.mapdepth = PRTE_BIND_TO_L1CACHE; options.maptype = HWLOC_OBJ_L1CACHE; break; case PRTE_MAPPING_BYCORE: if (1 < options.cpus_per_rank && !options.use_hwthreads) { /* we cannot support this operation as there is only one * cpu in a core */ pmix_show_help("help-prte-rmaps-base.txt", "mapping-too-low", true, options.cpus_per_rank, 1, prte_rmaps_base_print_mapping(options.map)); jdata->exit_code = PRTE_ERR_SILENT; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } options.mapdepth = PRTE_BIND_TO_CORE; options.maptype = HWLOC_OBJ_CORE; break; case PRTE_MAPPING_BYHWTHREAD: if (1 < options.cpus_per_rank) { /* we cannot support this operation as there is only one * cpu in a core */ pmix_show_help("help-prte-rmaps-base.txt", "mapping-too-low", true, options.cpus_per_rank, 1, prte_rmaps_base_print_mapping(options.map)); jdata->exit_code = PRTE_ERR_SILENT; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } options.mapdepth = PRTE_BIND_TO_HWTHREAD; options.maptype = HWLOC_OBJ_PU; break; case PRTE_MAPPING_PPR: break; default: PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } if (options.userranked) { /* must rank by user */ PRTE_SET_RANKING_POLICY(jdata->map->ranking, PRTE_RANKING_BYUSER); } else { /* set the default ranking policy IFF it wasn't provided */ if (!PRTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) { did_map = false; if (inherit) { if (NULL != parent) { jdata->map->ranking = parent->map->ranking; did_map = true; } else if (PRTE_RANKING_GIVEN & PRTE_GET_RANKING_DIRECTIVE(prte_rmaps_base.ranking)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps ranking given by MCA param"); jdata->map->ranking = prte_rmaps_base.ranking; did_map = true; } } if (!did_map) { // let the job's personality set the default ranking behavior if (NULL != schizo->set_default_ranking) { rc = schizo->set_default_ranking(jdata, &options); } else { rc = prte_rmaps_base_set_default_ranking(jdata, &options); } if (PRTE_SUCCESS != rc) { // the error message should have been printed jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } } } options.rank = PRTE_GET_RANKING_POLICY(jdata->map->ranking); /* if we are ranking by FILL or SPAN, then we must map by an object */ if ((PRTE_RANK_BY_SPAN == options.rank || PRTE_RANK_BY_FILL == options.rank) && PRTE_MAPPING_PPR != options.map) { if (options.map < PRTE_MAPPING_BYNUMA || options.map > PRTE_MAPPING_BYHWTHREAD) { pmix_show_help("help-prte-rmaps-base.txt", "must-map-by-obj", true, prte_rmaps_base_print_mapping(options.map), prte_rmaps_base_print_ranking(options.rank)); jdata->exit_code = PRTE_ERR_SILENT; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } /* define the binding policy for this job - if the user specified one * already (e.g., during the call to comm_spawn), then we don't * override it */ if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { did_map = false; if (options.oversubscribe) { /* if we are oversubscribing, then do not bind */ jdata->map->binding = PRTE_BIND_TO_NONE; did_map = true; } else if (inherit) { if (NULL != parent) { jdata->map->binding = parent->map->binding; did_map = true; } else if (PRTE_BINDING_POLICY_IS_SET(prte_hwloc_default_binding_policy)) { /* if the user specified a default binding policy via * MCA param, then we use it - this can include a directive * to overload */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps[%d] default binding policy given", __LINE__); jdata->map->binding = prte_hwloc_default_binding_policy; did_map = true; } } if (!did_map) { // let the job's personality set the default binding behavior if (NULL != schizo->set_default_binding) { rc = schizo->set_default_binding(jdata, &options); } else { rc = prte_hwloc_base_set_default_binding(jdata, &options); } if (PRTE_SUCCESS != rc) { // the error message should have been printed jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } } options.overload = PRTE_BIND_OVERLOAD_ALLOWED(jdata->map->binding); options.bind = PRTE_GET_BINDING_POLICY(jdata->map->binding); /* sanity check */ if (options.mapdepth > options.bind && PRTE_BIND_TO_NONE != options.bind) { /* we cannot bind to objects higher in the * topology than where we mapped */ pmix_show_help("help-prte-hwloc-base.txt", "bind-upwards", true, prte_rmaps_base_print_mapping(options.map), prte_hwloc_base_print_binding(options.bind)); jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } switch (options.bind) { case PRTE_BIND_TO_NONE: options.hwb = HWLOC_OBJ_MACHINE; break; case PRTE_BIND_TO_PACKAGE: options.hwb = HWLOC_OBJ_PACKAGE; break; case PRTE_BIND_TO_NUMA: options.hwb = HWLOC_OBJ_NUMANODE; break; case PRTE_BIND_TO_L3CACHE: options.hwb = HWLOC_OBJ_L3CACHE; break; case PRTE_BIND_TO_L2CACHE: options.hwb = HWLOC_OBJ_L2CACHE; break; case PRTE_BIND_TO_L1CACHE: options.hwb = HWLOC_OBJ_L1CACHE; break; case PRTE_BIND_TO_CORE: options.hwb = HWLOC_OBJ_CORE; break; case PRTE_BIND_TO_HWTHREAD: options.hwb = HWLOC_OBJ_PU; break; default: PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } if (1 < options.cpus_per_rank || NULL != options.job_cpuset || options.ordered) { /* REQUIRES binding to cpu */ if (PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { if (PRTE_BIND_TO_CORE != options.bind && PRTE_BIND_TO_HWTHREAD != options.bind) { pmix_show_help("help-prte-rmaps-base.txt", "unsupported-combination", true, "binding", prte_hwloc_base_print_binding(options.bind)); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* ensure the cpu usage setting matches the provided bind directive */ if (PRTE_BIND_TO_HWTHREAD == options.bind) { options.use_hwthreads = true; } else { options.use_hwthreads = false; } } else { if (options.use_hwthreads) { PRTE_SET_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_HWTHREAD); options.bind = PRTE_BIND_TO_HWTHREAD; } else { PRTE_SET_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_CORE); options.bind = PRTE_BIND_TO_CORE; } } } /* if we are not going to launch, then we need to set any * undefined topologies to match our own so the mapper * can operate */ if (options.donotlaunch) { prte_topology_t *t0; if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, 0))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PMIX_RELEASE(caddy); jdata->exit_code = PRTE_ERR_NOT_FOUND; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } t0 = node->topology; for (int i = 1; i < prte_node_pool->size; i++) { if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, i))) { continue; } if (NULL == node->topology) { node->topology = t0; } } } if (colocate_daemons || colocate) { /* This is a colocation request, so we don't run any mapping modules */ if (procs_per_target == 0) { pmix_output(0, "Error: COLOCATION REQUESTED WITH ZERO PROCS/TARGET\n"); jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } rc = map_colocate(jdata, colocate_daemons, pernode, darray, procs_per_target, &options); PMIX_DATA_ARRAY_FREE(darray); if (PRTE_SUCCESS != rc) { jdata->exit_code = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(jdata->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } did_map = true; } else { /* cycle thru the available mappers until one agrees to map * the job */ did_map = false; if (1 == pmix_list_get_size(&prte_rmaps_base.selected_modules)) { /* forced selection */ mod = (prte_rmaps_base_selected_module_t *) pmix_list_get_first( &prte_rmaps_base.selected_modules); jdata->map->req_mapper = strdup(mod->component->pmix_mca_component_name); } PMIX_LIST_FOREACH(mod, &prte_rmaps_base.selected_modules, prte_rmaps_base_selected_module_t) { if (PRTE_SUCCESS == (rc = mod->module->map_job(jdata, &options)) || PRTE_ERR_RESOURCE_BUSY == rc) { did_map = true; break; } /* mappers return "next option" if they didn't attempt to * map the job. anything else is a true error. */ if (PRTE_ERR_TAKE_NEXT_OPTION != rc) { jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } } } if (did_map && PRTE_ERR_RESOURCE_BUSY == rc) { /* the map was done but nothing could be mapped * for launch as all the resources were busy */ pmix_show_help("help-prte-rmaps-base.txt", "cannot-launch", true); jdata->exit_code = rc; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* if we get here without doing the map, or with zero procs in * the map, then that's an error */ if (!did_map || 0 == jdata->num_procs || 0 == jdata->map->num_nodes) { pmix_show_help("help-prte-rmaps-base.txt", "failed-map", true, PRTE_ERROR_NAME(rc), "N/A", jdata->num_procs, prte_rmaps_base_print_mapping(options.map), prte_hwloc_base_print_binding(options.bind)); jdata->exit_code = -PRTE_JOB_STATE_MAP_FAILED; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_FAILED); goto cleanup; } /* set the offset so shared memory components can potentially * connect to any spawned jobs */ jdata->offset = prte_total_procs; /* track the total number of procs launched by us */ prte_total_procs += jdata->num_procs; /* if it is a dynamic spawn, save the bookmark on the parent's job too */ if (!PMIX_NSPACE_INVALID(jdata->originator.nspace)) { if (NULL != (parent = prte_get_job_data_object(jdata->originator.nspace))) { parent->bookmark = jdata->bookmark; } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_MAP, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL)) { /* display the map */ prte_rmaps_base_display_map(jdata); } else if (options.donotlaunch && prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_BINDINGS, NULL, PMIX_BOOL)) { prte_rmaps_base_report_bindings(jdata, &options); } /* set the job state to the next position */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP_COMPLETE); cleanup: /* reset any node map flags we used so the next job will start clean */ for (int i = 0; i < jdata->map->nodes->size; i++) { if (NULL != (node = (prte_node_t *) pmix_pointer_array_get_item(jdata->map->nodes, i))) { PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } } if (NULL != options.job_cpuset) { hwloc_bitmap_free(options.job_cpuset); options.job_cpuset = NULL; } if (NULL != options.target) { hwloc_bitmap_free(options.target); options.target = NULL; } /* cleanup */ PMIX_RELEASE(caddy); } void prte_rmaps_base_display_map(prte_job_t *jdata) { pmix_proc_t source; char *tmp; prte_map_print(&tmp, jdata); PMIX_LOAD_PROCID(&source, jdata->nspace, PMIX_RANK_WILDCARD); prte_iof_base_output(&source, PMIX_FWD_STDOUT_CHANNEL, tmp); } void prte_rmaps_base_report_bindings(prte_job_t *jdata, prte_rmaps_options_t *options) { int n; prte_proc_t *proc; char **cache = NULL; char *out, *tmp; pmix_proc_t source; bool physical; // see if we are to report physical (vs logical) cpu IDs physical = prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_PHYSICAL_CPUS, NULL, PMIX_BOOL); for (n=0; n < jdata->procs->size; n++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, n); if (NULL == proc) { continue; } if (NULL == proc->cpuset) { pmix_asprintf(&out, "Proc %s Node %s is UNBOUND", PRTE_NAME_PRINT(&proc->name), proc->node->name); } else { hwloc_bitmap_list_sscanf(prte_rmaps_base.available, proc->cpuset); tmp = prte_hwloc_base_cset2str(prte_rmaps_base.available, options->use_hwthreads, physical, proc->node->topology->topo); pmix_asprintf(&out, "Proc %s Node %s bound to %s", PRTE_NAME_PRINT(&proc->name), proc->node->name, tmp); free(tmp); } PMIx_Argv_append_nosize(&cache, out); free(out); } if (NULL == cache) { out = strdup("Error: job has no procs"); } else { /* add a blank line with \n on it so IOF will output the last line */ PMIx_Argv_append_nosize(&cache, ""); out = PMIx_Argv_join(cache, '\n'); PMIx_Argv_free(cache); } PMIX_LOAD_PROCID(&source, jdata->nspace, PMIX_RANK_WILDCARD); prte_iof_base_output(&source, PMIX_FWD_STDOUT_CHANNEL, out); } static int map_colocate(prte_job_t *jdata, bool daemons, bool pernode, pmix_data_array_t *darray, uint16_t procs_per_target, prte_rmaps_options_t *options) { char *tmp; pmix_status_t rc; size_t n, nprocs; pmix_proc_t *procs; prte_job_t *target_jdata; prte_job_map_t *target_map, *map; prte_app_context_t *app; int i, j, ret, cnt; pmix_list_t targets; prte_proc_t *proc; prte_node_t *node, *nptr, *n2; if (4 < pmix_output_get_verbosity(prte_rmaps_base_framework.framework_output)) { rc = PMIx_Data_print(&tmp, NULL, darray, PMIX_DATA_ARRAY); if (PMIX_SUCCESS != rc) { pmix_output(0, "%s rmaps: mapping job %s: Colocate with UNPRINTABLE (%s)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), PMIx_Error_string(rc)); } else { pmix_output(0, "%s rmaps: mapping job %s: Colocate with\n %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), tmp); } free(tmp); } procs = (pmix_proc_t*)darray->array; nprocs = darray->size; map = jdata->map; if (daemons) { /* daemons are never bound and always rank by-slot */ PRTE_SET_BINDING_POLICY(map->binding, PRTE_BIND_TO_NONE); PRTE_SET_RANKING_POLICY(map->ranking, PRTE_RANK_BY_SLOT); } jdata->num_procs = 0; /* create a list of the target nodes */ PMIX_CONSTRUCT(&targets, pmix_list_t); /* need to ensure each node only appears _once_ on the list */ for (n=0; n < nprocs; n++) { if (PMIX_RANK_WILDCARD == procs[n].rank) { target_jdata = prte_get_job_data_object(procs[n].nspace); if (NULL == target_jdata) { pmix_output(0, "Unable to find app job %s\n", procs[n].nspace); ret = PRTE_ERR_BAD_PARAM; goto done; } target_map = target_jdata->map; for (i = 0; i < target_map->nodes->size; i++) { node = (prte_node_t*)pmix_pointer_array_get_item(target_map->nodes, i); if (NULL == node) { continue; } if (!PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_MAPPED)) { PRTE_FLAG_SET(node, PRTE_NODE_FLAG_MAPPED); PMIX_RETAIN(node); pmix_list_append(&targets, &node->super); } } continue; } /* not a wildcard rank */ proc = prte_get_proc_object(&procs[n]); if (NULL == proc) { pmix_output(0, "Unable to find target process %s\n", PMIX_NAME_PRINT(&procs[n])); ret = PRTE_ERR_BAD_PARAM; goto done; } if (NULL == proc->node) { pmix_output(0, "Target process %s has not been mapped to a node\n", PMIX_NAME_PRINT(&procs[n])); ret = PRTE_ERR_BAD_PARAM; goto done; } node = proc->node; if (!PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_MAPPED)) { /* add this node to the list */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_MAPPED); PMIX_RETAIN(node); pmix_list_append(&targets, &node->super); } } // clear the flags PMIX_LIST_FOREACH(nptr, &targets, prte_node_t) { PRTE_FLAG_UNSET(nptr, PRTE_NODE_FLAG_MAPPED); } if (pernode) { /* cycle across the target nodes and place the specified * number of procs on each one */ PMIX_LIST_FOREACH_SAFE(nptr, n2, &targets, prte_node_t) { // setup the mapping options options->ncpus = prte_rmaps_base_get_ncpus(nptr, NULL, options); /* the available cpus are in the scratch location */ options->target = hwloc_bitmap_dup(prte_rmaps_base.available); options->nprocs = procs_per_target; // Assign N procs per node for each app_context for (i=0; i < jdata->apps->size; i++) { app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } // is there room on this node? daemons don't count if (!daemons) { cnt = nptr->slots_inuse + procs_per_target; // first check the absolute limit if (0 != nptr->slots_max && nptr->slots_max < cnt) { // violates the max limit ret = PRTE_ERR_OUT_OF_RESOURCE; goto done; } if (nptr->slots < cnt) { // oversubscribed - we can still fit if they allow oversubscription if (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(map->mapping)) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); ret = PRTE_ERR_SILENT; goto done; } // we can use it oversubscribed - so mark it PRTE_FLAG_SET(nptr, PRTE_NODE_FLAG_OVERSUBSCRIBED); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_OVERSUBSCRIBED); } } // Map the node to this job if (!PRTE_FLAG_TEST(nptr, PRTE_NODE_FLAG_MAPPED)) { PRTE_FLAG_SET(nptr, PRTE_NODE_FLAG_MAPPED); PMIX_RETAIN(nptr); pmix_pointer_array_add(map->nodes, nptr); map->num_nodes += 1; options->nnodes++; } // map the procs for (j = 0; j < procs_per_target; ++j) { proc = prte_rmaps_base_setup_proc(jdata, app->idx, nptr, NULL, options); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); ret = PRTE_ERR_OUT_OF_RESOURCE; goto done; } jdata->num_procs += 1; app->num_procs += 1; PMIX_RELEASE(proc); } } } /* calculate the ranks for this job */ ret = prte_rmaps_base_compute_vpids(jdata, options); if (PRTE_SUCCESS != ret) { return ret; } ret = PRTE_SUCCESS; goto done; } /* handle the case of colocate by process */ PMIX_LIST_FOREACH_SAFE(nptr, n2, &targets, prte_node_t) { // setup the mapping options options->ncpus = prte_rmaps_base_get_ncpus(nptr, NULL, options); /* the available cpus are in the scratch location */ options->target = hwloc_bitmap_dup(prte_rmaps_base.available); // count the number of target procs on this node cnt = 0; for (i=0; i < nptr->procs->size; i++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(nptr->procs, i); if (NULL == proc) { continue; } for (n=0; n < nprocs; n++) { if (PMIX_CHECK_PROCID(&procs[n], &proc->name)) { ++cnt; break; } } } if (0 == cnt) { // should not happen continue; } // assign the number of procs to be placed on this node options->nprocs = cnt * procs_per_target; // total number of procs to place on this node; // Assign procs for each app_context for (i=0; i < jdata->apps->size; i++) { app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } // is there room on this node? daemons don't count if (!daemons) { cnt = nptr->slots_inuse + options->nprocs; // first check the absolute limit if (0 != nptr->slots_max && nptr->slots_max < cnt) { // violates the max limit ret = PRTE_ERR_OUT_OF_RESOURCE; goto done; } // check oversubscribed if (nptr->slots < cnt) { // oversubscribed - we can still fit if they allow oversubscription if (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(map->mapping)) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); ret = PRTE_ERR_SILENT; goto done; } // we can use it oversubscribed - so mark it PRTE_FLAG_SET(nptr, PRTE_NODE_FLAG_OVERSUBSCRIBED); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_OVERSUBSCRIBED); } } // Map the node to this job if (!PRTE_FLAG_TEST(nptr, PRTE_NODE_FLAG_MAPPED)) { PRTE_FLAG_SET(nptr, PRTE_NODE_FLAG_MAPPED); PMIX_RETAIN(nptr); pmix_pointer_array_add(map->nodes, nptr); map->num_nodes += 1; options->nnodes++; } // map the procs for (j = 0; j < options->nprocs; ++j) { proc = prte_rmaps_base_setup_proc(jdata, i, nptr, NULL, options); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); ret = PRTE_ERR_OUT_OF_RESOURCE; goto done; } jdata->num_procs += 1; app->num_procs += 1; PMIX_RELEASE(proc); } } } ret = prte_rmaps_base_compute_vpids(jdata, options); if (PRTE_SUCCESS != ret) { return ret; } ret = PRTE_SUCCESS; done: // ensure all the nodes are marked as not mapped for (i=0; i < map->nodes->size; i++) { node = (prte_node_t*)pmix_pointer_array_get_item(map->nodes, i); if (NULL != node) { PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } } PMIX_LIST_DESTRUCT(&targets); return ret; } static void inherit_env_directives(prte_job_t *jdata, prte_job_t *parent, pmix_proc_t *proxy) { prte_app_context_t *app, *app2; prte_proc_t *p; prte_attribute_t *attr, *attr2; pmix_value_t *val, *val2; pmix_envar_t *envar, *envar2; int n; bool exists; // deal with job-level attributes first PMIX_LIST_FOREACH(attr, &parent->attributes, prte_attribute_t) { if (PMIX_ENVAR != attr->data.type) { continue; } val = &attr->data; envar = &val->data.envar; // do we have a matching attribute in the new job? exists = false; PMIX_LIST_FOREACH(attr2, &jdata->attributes, prte_attribute_t) { if (PMIX_ENVAR != attr->data.type) { continue; } val2 = &attr2->data; envar2 = &val2->data.envar; if (attr->key == attr2->key) { // operation is same - check if the target envars match if (0 == strcmp(envar->envar, envar2->envar)) { // these match, so don't overwrite it exists = true; break; } } } if (exists) { // leave this alone continue; } // if it doesn't exist, then inherit it prte_set_attribute(&jdata->attributes, attr->key, PRTE_ATTR_GLOBAL, envar, PMIX_ENVAR); } /* There is no one-to-one correlation between the apps, but we can * inherit the directives from the proc that called spawn, so do that * much here */ p = prte_get_proc_object(proxy); if (NULL == p) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return; } app = (prte_app_context_t*)pmix_pointer_array_get_item(parent->apps, p->app_idx); if (NULL == app) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return; } for (n=0; n < jdata->apps->size; n++) { app2 = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app2) { continue; } PMIX_LIST_FOREACH(attr, &app->attributes, prte_attribute_t) { if (PMIX_ENVAR != attr->data.type) { continue; } val = &attr->data; envar = &val->data.envar; exists = false; PMIX_LIST_FOREACH(attr2, &app2->attributes, prte_attribute_t) { if (PMIX_ENVAR != attr->data.type) { continue; } val2 = &attr2->data; envar2 = &val2->data.envar; if (attr->key == attr2->key) { // operation is same - check if the target envars match if (0 == strcmp(envar->envar, envar2->envar)) { // these match, so don't overwrite it exists = true; break; } } } if (exists) { // leave this alone continue; } // if it doesn't exist, then inherit it prte_set_attribute(&app2->attributes, attr->key, PRTE_ATTR_GLOBAL, envar, PMIX_ENVAR); } } } prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_select.c0000664000175000017500000001115515203542132023044 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/rmaps/base/base.h" static bool selected = false; /* * Function for selecting one component from all those that are * available. */ int prte_rmaps_base_select(void) { pmix_mca_base_component_list_item_t *cli = NULL; pmix_mca_base_component_t *component = NULL; pmix_mca_base_module_t *module = NULL; prte_rmaps_base_module_t *nmodule; prte_rmaps_base_selected_module_t *newmodule, *mod; int rc, priority; bool inserted; if (selected) { /* ensure we don't do this twice */ return PRTE_SUCCESS; } selected = true; /* Query all available components and ask if they have a module */ PMIX_LIST_FOREACH(cli, &prte_rmaps_base_framework.framework_components, pmix_mca_base_component_list_item_t) { component = (pmix_mca_base_component_t *) cli->cli_component; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:select: checking available component %s", component->pmix_mca_component_name); /* If there's no query function, skip it */ if (NULL == component->pmix_mca_query_component) { pmix_output_verbose( 5, prte_rmaps_base_framework.framework_output, "mca:rmaps:select: Skipping component [%s]. It does not implement a query function", component->pmix_mca_component_name); continue; } /* Query the component */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:select: Querying component [%s]", component->pmix_mca_component_name); rc = component->pmix_mca_query_component(&module, &priority); /* If no module was returned, then skip component */ if (PRTE_SUCCESS != rc || NULL == module) { pmix_output_verbose( 5, prte_rmaps_base_framework.framework_output, "mca:rmaps:select: Skipping component [%s]. Query failed to return a module", component->pmix_mca_component_name); continue; } /* If we got a module, keep it */ nmodule = (prte_rmaps_base_module_t *) module; /* add to the list of selected modules */ newmodule = PMIX_NEW(prte_rmaps_base_selected_module_t); newmodule->pri = priority; newmodule->module = nmodule; newmodule->component = component; /* maintain priority order */ inserted = false; PMIX_LIST_FOREACH(mod, &prte_rmaps_base.selected_modules, prte_rmaps_base_selected_module_t) { if (priority > mod->pri) { pmix_list_insert_pos(&prte_rmaps_base.selected_modules, (pmix_list_item_t *) mod, &newmodule->super); inserted = true; break; } } if (!inserted) { /* must be lowest priority - add to end */ pmix_list_append(&prte_rmaps_base.selected_modules, &newmodule->super); } } if (4 < pmix_output_get_verbosity(prte_rmaps_base_framework.framework_output)) { pmix_output(0, "%s: Final mapper priorities", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); /* show the prioritized list */ PMIX_LIST_FOREACH(mod, &prte_rmaps_base.selected_modules, prte_rmaps_base_selected_module_t) { pmix_output(0, "\tMapper: %s Priority: %d", mod->component->pmix_mca_component_name, mod->pri); } } return PRTE_SUCCESS; ; } prrte-4.1.1~rc2/src/mca/rmaps/base/help-rankby.txt0000664000175000017500000000662415203542132022207 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRRTE's prun. # [rank-by] PRRTE automatically ranks processes for each job starting from zero. Regardless of the algorithm used, rank assignments span applications in the same job — i.e., a command line of -n 3 app1 : -n 2 app2 will result in "app1" having three processes ranked 0-2 and "app2" having two processes ranked 3-4. By default, process ranks are assigned in accordance with the mapping directive — e.g., jobs that are mapped by-node will have the process ranks assigned round-robin on a per-node basis. However, users can override the default by specifying any of the following directives using the "--rank-by" command line option: * "SLOT" assigns ranks to each process on a node in the order in which the mapper assigned them. This is the default behavior, but is provided as an explicit option to allow users to override any alternative default specified in the environment. When mapping to a specific resource type, procs assigned to a given instance of that resource on a node will be ranked on a per-resource basis on that node before moving to the next node. * "NODE" assigns ranks round-robin on a per-node basis * "FILL" assigns ranks to procs mapped to a particular resource type on each node, filling all ranks on that resource before moving to the next resource on that node. For example, procs mapped by "L1cache" would have all procs on the first "L1cache" ranked sequentially before moving to the second "L1cache" on the node. Once all procs on the node have been ranked, ranking would continue on the next node. * "SPAN" assigns ranks round-robin to procs mapped to a particular resource type, treating the collection of resource instances spanning the entire allocation as a single "super node" before looping around for the next pass. Thus, ranking would begin with the first proc on the first "L1cache" on the first node, then the next rank would be assigned to the first proc on the second "L1cache" on that node, proceeding across until the first proc had been ranked on all "L1cache" used by the job before circling around to rank the second proc on each object. The "rank-by" command line option has no qualifiers. Note: Directives are case-insensitive. "SPAN" is the same as "span". A more detailed description of the mapping, ranking, and binding procedure can be obtained via the "--help placement" option. prrte-4.1.1~rc2/src/mca/rmaps/base/help-placement.txt0000664000175000017500000013726215203542132022674 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRRTE's prun. # [placement] Overview -------- PRRTE provides a set of three controls for assigning process locations and ranks: 1. Mapping: Assigns a default location to each process 2. Ranking: Assigns a unique integer rank value to each process 3. Binding: Constrains each process to run on specific processors This section provides an overview of these three controls. Unless otherwise this behavior is shared by "prun(1)" (working with a PRRTE DVM), and "prterun(1)". More detail about PRRTE process placement is available in the following sections (using "--help placement-
"): * "examples": some examples of the interactions between mapping, ranking, and binding options. * "fundamentals": provides deeper insight into PRRTE's mapping, ranking, and binding options. * "limits": explains the difference between *overloading* and *oversubscribing* resources. * "diagnostics": describes options for obtaining various diagnostic reports that aid the user in verifying and tuning the placement for a specific job. * "rankfiles": explains the format and use of the rankfile mapper for specifying arbitrary process placements. * "deprecated": a list of deprecated options and their new equivalents. * "all": outputs all the placement help except for the "deprecated" and "rankfile" sections. Quick Summary ============= The two binaries that most influence process layout are "prte(1)" and "prun(1)". The "prte(1)" process discovers the allocation, establishes a Distributed Virtual Machine by starting a "prted(1)" daemon on each node of the allocation, and defines the efault mapping/ranking/binding policies for all jobs. The "prun(1)" process defines the specific mapping/ranking/binding for a specific job. Most of the command line controls are targeted to "prun(1)" since each job has its own unique requirements. "prterun(1)" is just a wrapper around "prte(1)" for a single job PRRTE DVM. It is doing the job of both "prte(1)" and "prun(1)", and, as such, accepts the sum all of their command line arguments. Any example that uses "prun(1)" can substitute the use of "prterun(1)" except where otherwise noted. The "prte(1)" process attempts to automatically discover the nodes in the allocation by querying supported resource managers. If a supported resource manager is not present then "prte(1)" relies on a hostfile provided by the user. In the absence of such a hostfile it will run all processes on the localhost. If running under a supported resource manager, the "prte(1)" process will start the daemon processes ("prted(1)") on the remote nodes using the corresponding resource manager process starter. If no such starter is available then "ssh" (or "rsh") is used. Minus user direction, PRRTE will automatically map processes in a round-robin fashion by CPU, binding each process to its own CPU. The type of CPU used (core vs hwthread) is determined by (in priority order): * user directive on the command line via the HWTCPUS qualifier to the "--map-by" directive * setting the "rmaps_default_mapping_policy" MCA parameter to include the "HWTCPUS" qualifier. This parameter sets the default value for a PRRTE DVM — qualifiers are carried across to DVM jobs started via "prun" unless overridden by the user's command line * defaulting to "CORE" in topologies where core CPUs are defined, and to "hwthreads" otherwise. By default, the ranks are assigned in accordance with the mapping directive — e.g., jobs that are mapped by-node will have the process ranks assigned round-robin on a per-node basis. PRRTE automatically binds processes unless directed not to do so by the user. Minus direction, PRRTE will bind individual processes to their own CPU within the object to which they were mapped. Should a node become oversubscribed during the mapping process, and if oversubscription is allowed, all subsequent processes assigned to that node will *not* be bound. Definition of 'slot' -------------------- The term "slot" is used extensively in the rest of this documentation. A slot is an allocation unit for a process. The number of slots on a node indicate how many processes can potentially execute on that node. By default, PRRTE will allow one process per slot. If PRRTE is not explicitly told how many slots are available on a node (e.g., if a hostfile is used and the number of slots is not specified for a given node), it will determine a maximum number of slots for that node in one of two ways: 1. Default behavior: By default, PRRTE will attempt to discover the number of processor cores on the node, and use that as the number of slots available. 2. When "--use-hwthread-cpus" is used: If "--use-hwthread-cpus" is specified on the command line, then PRRTE will attempt to discover the number of hardware threads on the node, and use that as the number of slots available. This default behavior also occurs when specifying the "--host" option with a single host. Thus, the command: shell$ prun --host node1 ./a.out launches a number of processes equal to the number of cores on node "node1", whereas: shell$ prun --host node1 --use-hwthread-cpus ./a.out launches a number of processes equal to the number of hardware threads on "node1". When PRRTE applications are invoked in an environment managed by a resource manager (e.g., inside of a Slurm job), and PRRTE was built with appropriate support for that resource manager, then PRRTE will be informed of the number of slots for each node by the resource manager. For example: shell$ prun ./a.out launches one process for every slot (on every node) as dictated by the resource manager job specification. Also note that the one-process-per-slot restriction can be overridden in unmanaged environments (e.g., when using hostfiles without a resource manager) if oversubscription is enabled (by default, it is disabled). Most parallel applications and HPC environments do not oversubscribe; for simplicity, the majority of this documentation assumes that oversubscription is not enabled. Slots are not hardware resources ================================ Slots are frequently incorrectly conflated with hardware resources. It is important to realize that slots are an entirely different metric than the number (and type) of hardware resources available. Here are some examples that may help illustrate the difference: 1. More processor cores than slots: Consider a resource manager job environment that tells PRRTE that there is a single node with 20 processor cores and 2 slots available. By default, PRRTE will only let you run up to 2 processes. Meaning: you run out of slots long before you run out of processor cores. 2. More slots than processor cores: Consider a hostfile with a single node listed with a "slots=50" qualification. The node has 20 processor cores. By default, PRRTE will let you run up to 50 processes. Meaning: you can run many more processes than you have processor cores. Definition of "processor element" --------------------------------- By default, PRRTE defines that a "processing element" is a processor core. However, if "--use-hwthread-cpus" is specified on the command line, then a "processing element" is a hardware thread. # [placement-all] #include#help-placement.txt#placement-fundamentals #include#help-placement.txt#placement-limits #include#help-placement.txt#placement-examples #include#help-placement.txt#placement-diagnostics # [placement-examples] Examples -------- Listed here are the subset of command line options that will be used in the process mapping/ranking/binding examples below. Specifying Host Nodes ===================== Use one of the following options to specify which hosts (nodes) within the PRRTE DVM environment to run on. --host # or --host * List of hosts on which to invoke processes. After each hostname a colon (":") followed by a positive integer can be used to specify the number of slots on that host (":X", ":Y", and ":Z"). The default is "1". --hostfile * Provide a hostfile to use. Process Mapping / Ranking / Binding Options =========================================== * "-c #", "-n #", "--n #", "--np <#>": Run this many copies of the program on the given nodes. This option indicates that the specified file is an executable program and not an application context. If no value is provided for the number of copies to execute (i.e., neither the "-np" nor its synonyms are provided on the command line), "prun" will automatically execute a copy of the program on each process slot (see below for description of a "process slot"). This feature, however, can only be used in the SPMD model and will return an error (without beginning execution of the application) otherwise. Note: These options specify the number of processes to launch. None of the options imply a particular binding policy — e.g., requesting "N" processes for each package does not imply that the processes will be bound to the package. * "--map-by ": Map to the specified object. Supported objects include: * "slot" * "hwthread" * "core" (default) * "l1cache" * "l2cache" * "l3cache" * "numa" * "package" * "node" * "seq" * "ppr" * "rankfile" * "pe-list" Any object can include qualifiers by adding a colon (":") and any colon-delimited combination of one or more of the following to the " --map-by" options: * "PE=n" bind "n" processing elements to each process (can not be used in combination with rankfile or pe-list directives) Error: JMS Several of the options below refer to "pe-list". Is this option supposed to be "PE-LIST=n", not "PE=n"? * "SPAN" load balance the processes across the allocation (cannot be used in combination with "slot", "node", "seq", "ppr", "rankfile", or "pe-list" directives) * "OVERSUBSCRIBE" allow more processes on a node than processing elements * "NOOVERSUBSCRIBE" means "!OVERSUBSCRIBE" * "NOLOCAL" do not launch processes on the same node as "prun" * "HWTCPUS" use hardware threads as CPU slots * "CORECPUS" use cores as CPU slots (default) * "INHERIT" indicates that a child job (i.e., one spawned from within an application) shall inherit the placement policies of the parent job that spawned it. * "NOINHERIT" means "!INHERIT" * "FILE=" (path to file containing sequential or rankfile entries). * "ORDERED" only applies to the PE-LIST option to indicate that procs are to be bound to each of the specified CPUs in the order in which they are assigned (i.e., the first proc on a node shall be bound to the first CPU in the list, the second proc shall be bound to the second CPU, etc.) "ppr" policy example: "--map-by ppr:N:" will launch "N" times the number of objects of the specified type on each node. Note: Directives and qualifiers are case-insensitive and can be shortened to the minimum number of characters to uniquely identify them. Thus, "L1CACHE" can be given as "l1cache" or simply as "L1". * "--rank-by ": This assigns ranks in round-robin fashion according to the specified object. The default follows the mapping pattern. Supported rank-by objects include: * "slot" * "node" * "fill" * "span" There are no qualifiers for the "--rank-by" directive. * "--bind-to ": This binds processes to the specified object. See defaults in Quick Summary. Supported bind-to objects include: * "none" * "hwthread" * "core" * "l1cache" * "l2cache" * "l3cache" * "numa" * "package" Any object can include qualifiers by adding a colon (":") and any colon-delimited combination of one or more of the following to the " --bind-to" options: * "overload-allowed" allows for binding more than one process in relation to a CPU * "if-supported" if binding to that object is supported on this system. Specifying Host Nodes ===================== Host nodes can be identified on the command line with the "--host" option or in a hostfile. For example, assuming no other resource manager or scheduler is involved: prun --host aa,aa,bb ./a.out This launches two processes on node "aa" and one on "bb". prun --host aa ./a.out This launches one process on node "aa". prun --host aa:5 ./a.out This launches five processes on node "aa". Or, consider the hostfile: $ cat myhostfile aa slots=2 bb slots=2 cc slots=2 Here, we list both the host names ("aa", "bb", and "cc") but also how many "slots" there are for each. Slots indicate how many processes can potentially execute on a node. For best performance, the number of slots may be chosen to be the number of cores on the node or the number of processor sockets. If the hostfile does not provide slots information, the PRRTE DVM will attempt to discover the number of cores (or hwthreads, if the ":HWTCPUS" qualifier to the "--map-by" option is set) and set the number of slots to that value. Examples using the hostfile above with and without the "--host" option: prun --hostfile myhostfile ./a.out This will launch two processes on each of the three nodes. prun --hostfile myhostfile --host aa ./a.out This will launch two processes, both on node "aa". prun --hostfile myhostfile --host dd ./a.out This will find no hosts to run on and abort with an error. That is, the specified host "dd" is not in the specified hostfile. When running under resource managers (e.g., SLURM, Torque, etc.), PRTE will obtain both the hostnames and the number of slots directly from the resource manger. The behavior of "--host" in that environment will behave the same as if a hostfile was provided (since it is provided by the resource manager). Specifying Number of Processes ============================== As we have just seen, the number of processes to run can be set using the hostfile. Other mechanisms exist. The number of processes launched can be specified as a multiple of the number of nodes or processor sockets available. Consider the hostfile below for the examples that follow. $ cat myhostfile aa bb For example: prun --hostfile myhostfile --map-by ppr:2:package ./a.out This launches processes 0-3 on node "aa" and process 4-7 on node "bb", where "aa" and "bb" are both dual-package nodes. The "--map-by ppr:2:package" option also turns on the "--bind-to package" option, which is discussed in a later section. prun --hostfile myhostfile --map-by ppr:2:node ./a.out This launches processes 0-1 on node "aa" and processes 2-3 on node "bb". prun --hostfile myhostfile --map-by ppr:1:node ./a.out This launches one process per host node. Another alternative is to specify the number of processes with the "-- np" option. Consider now the hostfile: $ cat myhostfile aa slots=4 bb slots=4 cc slots=4 With this hostfile: prun --hostfile myhostfile --np 6 ./a.out This will launch processes 0-3 on node "aa" and processes 4-5 on node "bb". The remaining slots in the hostfile will not be used since the "-np" option indicated that only 6 processes should be launched. Mapping Processes to Nodes Using Policies ========================================= The examples above illustrate the default mapping of process processes to nodes. This mapping can also be controlled with various "prun" / "prterun" options that describe mapping policies. $ cat myhostfile aa slots=4 bb slots=4 cc slots=4 Consider the hostfile above, with "--np 6": +---------------------------+---------------------------+---------------------------+---------------------------+ | Command | Ranks on "aa" | Ranks on "bb" | Ranks on "cc" | |===========================|===========================|===========================|===========================| | "prun" | 0 1 2 3 | 4 5 | | +---------------------------+---------------------------+---------------------------+---------------------------+ | "prun --map-by node" | 0 3 | 1 4 | 2 5 | +---------------------------+---------------------------+---------------------------+---------------------------+ | "prun --map-by | | 0 2 4 | 1 3 5 | | node:NOLOCAL" | | | | +---------------------------+---------------------------+---------------------------+---------------------------+ The "--map-by node" option will load balance the processes across the available nodes, numbering each process by node in a round-robin fashion. The ":NOLOCAL" qualifier to "--map-by" prevents any processes from being mapped onto the local host (in this case node "aa"). While "prun" typically consumes few system resources, the ":NOLOCAL" qualifier can be helpful for launching very large jobs where "prun" may actually need to use noticeable amounts of memory and/or processing time. Just as "--np" can specify fewer processes than there are slots, it can also oversubscribe the slots. For example, with the same hostfile: prun --hostfile myhostfile --np 14 ./a.out This will produce an error since the default ":NOOVERSUBSCRIBE" qualifier to "--map-by" prevents oversubscription. To oversubscribe the nodes you can use the ":OVERSUBSCRIBE" qualifier to "--map-by": prun --hostfile myhostfile --np 14 --map-by :OVERSUBSCRIBE ./a.out This will launch processes 0-5 on node "aa", 6-9 on "bb", and 10-13 on "cc". Limits to oversubscription can also be specified in the hostfile itself with the "max_slots" field: $ cat myhostfile aa slots=4 max_slots=4 bb max_slots=8 cc slots=4 The "max_slots" field specifies such a limit. When it does, the "slots" value defaults to the limit. Now: prun --hostfile myhostfile --np 14 --map-by :OVERSUBSCRIBE ./a.out This causes the first 12 processes to be launched as before, but the remaining two processes will be forced onto node cc. The other two nodes are protected by the hostfile against oversubscription by this job. Using the ":NOOVERSUBSCRIBE" qualifier to "--map-by" option can be helpful since the PRTE DVM currently does not get "max_slots" values from the resource manager. Of course, "--np" can also be used with the "--host" option. For example, prun --host aa,bb --np 8 ./a.out This will produce an error since the default ":NOOVERSUBSCRIBE" qualifier to "--map-by" prevents oversubscription. prun --host aa,bb --np 8 --map-by :OVERSUBSCRIBE ./a.out This launches 8 processes. Since only two hosts are specified, after the first two processes are mapped, one to "aa" and one to "bb", the remaining processes oversubscribe the specified hosts evenly. prun --host aa:2,bb:6 --np 8 ./a.out This launches 8 processes. Processes 0-1 on node "aa" since it has 2 slots and processes 2-7 on node "bb" since it has 6 slots. And here is a MIMD example: prun --host aa --np 1 hostname : --host bb,cc --np 2 uptime This will launch process 0 running "hostname" on node "aa" and processes 1 and 2 each running "uptime" on nodes "bb" and "cc", respectively. # [placement-rankfiles] Rankfiles --------- Another way to specify arbitrary mappings is with a rankfile, which gives you detailed control over process binding as well. Rankfiles are text files that specify detailed information about how individual processes should be mapped to nodes, and to which processor(s) they should be bound. Each line of a rankfile specifies the location of one process. The general form of each line in the rankfile is: rank = slot= For example: $ cat myrankfile rank 0=aa slot=10-12 rank 1=bb slot=0,1,4 rank 2=cc slot=1-2 $ prun --host aa,bb,cc,dd --map-by rankfile:FILE=myrankfile ./a.out Means that: * Rank 0 runs on node aa, bound to logical cores 10-12. * Rank 1 runs on node bb, bound to logical cores 0, 1, and 4. * Rank 2 runs on node cc, bound to logical cores 1 and 2. Similarly: $ cat myrankfile rank 0=aa slot=1:0-2 rank 1=bb slot=0:0,1,4 rank 2=cc slot=1-2 $ prun --host aa,bb,cc,dd --map-by rankfile:FILE=myrankfile ./a.out Means that: * Rank 0 runs on node aa, bound to logical package 1, cores 10-12 (the 0th through 2nd cores on that package). * Rank 1 runs on node bb, bound to logical package 0, cores 0, 1, and 4. * Rank 2 runs on node cc, bound to logical cores 1 and 2. The hostnames listed above are "absolute," meaning that actual resolvable hostnames are specified. However, hostnames can also be specified as "relative," meaning that they are specified in relation to an externally-specified list of hostnames (e.g., by "prun"'s "-- host" argument, a hostfile, or a job scheduler). The "relative" specification is of the form ""+n"", where "X" is an integer specifying the Xth hostname in the set of all available hostnames, indexed from 0. For example: $ cat myrankfile rank 0=+n0 slot=10-12 rank 1=+n1 slot=0,1,4 rank 2=+n2 slot=1-2 $ prun --host aa,bb,cc,dd --map-by rankfile:FILE=myrankfile ./a.out All package/core slot locations are be specified as *logical* indexes. You can use tools such as HWLOC's "lstopo" to find the logical indexes of packages and cores. # [placement-deprecated] Deprecated options ------------------ These deprecated options will be removed in a future release. +----------------------+----------------------+--------------------------------+ | Deprecated Option | Replacement | Description | |======================|======================|================================| | "--bind-to-core" | "--bind-to core" | Bind processes to cores | +----------------------+----------------------+--------------------------------+ | "--bind-to-socket" | "--bind-to package" | Bind processes to processor | | | | sockets | +----------------------+----------------------+--------------------------------+ | "--bycore" | "--map-by core" | Map processes by core | +----------------------+----------------------+--------------------------------+ | "--bynode" | "--map-by node" | Launch processes one per node, | | | | cycling by node in a round- | | | | robin fashion. This spreads | | | | processes evenly among nodes | | | | and assigns ranks in a round- | | | | robin, "by node" manner. | +----------------------+----------------------+--------------------------------+ | "--byslot" | "--map-by slot" | Map and rank processes round- | | | | robin by slot | +----------------------+----------------------+--------------------------------+ | "--cpus-per-proc | *--map-by :PE= | Bind each process to the | | <#perproc>" | <#perproc>`* | specified number of CPUs | +----------------------+----------------------+--------------------------------+ | "--cpus-per-rank | "--map-by | Alias for "--cpus-per-proc" | | <#perrank>" | :PE=<#perrank>" | | +----------------------+----------------------+--------------------------------+ | "--display- | "--display ALLOC" | Display the detected resource | | allocation" | | allocation | +----------------------+----------------------+--------------------------------+ | "-display-devel-map" | "--display MAP- | Display a detailed process map | | | DEVEL" | (mostly intended for | | | | developers) just before | | | | launch. | +----------------------+----------------------+--------------------------------+ | "--display-map" | "--display MAP" | Display a table showing the | | | | mapped location of each | | | | process prior to launch. | +----------------------+----------------------+--------------------------------+ | "--display-topo" | "--display TOPO" | Display the topology as part | | | | of the process map (mostly | | | | intended for developers) just | | | | before launch. | +----------------------+----------------------+--------------------------------+ | "--do-not-launch" | "--map-by | Perform all necessary | | | :DONOTLAUNCH" | operations to prepare to | | | | launch the application, but do | | | | not actually launch it | | | | (usually used to test mapping | | | | patterns). | +----------------------+----------------------+--------------------------------+ | "--do-not-resolve" | "--map-by | Do not attempt to resolve | | | :DONOTRESOLVE" | interfaces — usually used to | | | | determine proposed process | | | | placement/binding prior to | | | | obtaining an allocation. | +----------------------+----------------------+--------------------------------+ | "-N " | "--map-by | Launch "num" processes per | | | prr::node" | node on all allocated nodes | +----------------------+----------------------+--------------------------------+ | "--nolocal" | "--map-by :NOLOCAL" | Do not run any copies of the | | | | launched application on the | | | | same node as "prun" is | | | | running. This option will | | | | override listing the | | | | "localhost" with "--host" or | | | | any other host-specifying | | | | mechanism. | +----------------------+----------------------+--------------------------------+ | "--nooversubscribe" | "--map-by | Do not oversubscribe any | | | :NOOVERSUBSCRIBE" | nodes; error (without starting | | | | any processes) if the | | | | requested number of processes | | | | would cause oversubscription. | | | | This option implicitly sets | | | | "max_slots" equal to the | | | | "slots" value for each node. | | | | (Enabled by default). | +----------------------+----------------------+--------------------------------+ | "--npernode | "--map-by | On each node, launch this many | | <#pernode>" | ppr:<#pernode>:node" | processes | +----------------------+----------------------+--------------------------------+ | "--npersocket | "--map-by ppr:<#per | On each node, launch this many | | <#persocket>" | package>:package" | processes times the number of | | | | processor sockets on the node. | | | | The "--npersocket" option also | | | | turns on the "-- bind-to | | | | socket" option. The term | | | | "socket" has been globally | | | | replaced with "package". | +----------------------+----------------------+--------------------------------+ | "--oversubscribe" | "--map-by | Nodes are allowed to be | | | :OVERSUBSCRIBE" | oversubscribed, even on a | | | | managed system, and | | | | overloading of processing | | | | elements. | +----------------------+----------------------+--------------------------------+ | "--pernode" | "--map-by | On each node, launch one | | | ppr:1:node" | process | +----------------------+----------------------+--------------------------------+ | "--ppr" | *--map-by | Comma-separated list of number | | | ppr:`* | of processes on a given | | | | resource type [default: | | | | "none"]. | +----------------------+----------------------+--------------------------------+ | "--rankfile | "--map-by rankfile: | Use a rankfile for | | " | FILE=" | mapping/ranking/binding | +----------------------+----------------------+--------------------------------+ | "--report-bindings" | "--display BINDINGS" | Report any bindings for | | | | launched processes | +----------------------+----------------------+--------------------------------+ | "--tag-output" | "--output TAG" | Tag all output with | | | | "[job,rank]" | +----------------------+----------------------+--------------------------------+ | "--timestamp-output" | "--output TIMESTAMP" | Timestamp all application | | | | process output | +----------------------+----------------------+--------------------------------+ | "--use-hwthread- | "--map-by :HWTCPUS" | Use hardware threads as | | cpus" | | independent CPUs | +----------------------+----------------------+--------------------------------+ | "--xml" | "--output XML" | Provide all output in XML | | | | format | +----------------------+----------------------+--------------------------------+ # [placement-diagnostics] Diagnostics ----------- PRRTE provides various diagnostic reports that aid the user in verifying and tuning the mapping/ranking/binding for a specific job. The ":REPORT" qualifier to the "--bind-to" command line option can be used to report process bindings. As an example, consider a node with: * 2 processor packages, * 4 cores per package, and * 8 hardware threads per core. In each of the examples below the binding is reported in a human readable format. $ prun --np 4 --map-by core --bind-to core:REPORT ./a.out [node01:103137] MCW rank 0 bound to package[0][core:0] [node01:103137] MCW rank 1 bound to package[0][core:1] [node01:103137] MCW rank 2 bound to package[0][core:2] [node01:103137] MCW rank 3 bound to package[0][core:3] In the example above, processes are bound to successive cores on the first package. $ prun --np 4 --map-by package --bind-to package:REPORT ./a.out [node01:103115] MCW rank 0 bound to package[0][core:0-9] [node01:103115] MCW rank 1 bound to package[1][core:10-19] [node01:103115] MCW rank 2 bound to package[0][core:0-9] [node01:103115] MCW rank 3 bound to package[1][core:10-19] In the example above, processes are bound to all cores on successive packages in a round-robin fashion. $ prun --np 4 --map-by package:PE=2 --bind-to core:REPORT ./a.out [node01:103328] MCW rank 0 bound to package[0][core:0-1] [node01:103328] MCW rank 1 bound to package[1][core:10-11] [node01:103328] MCW rank 2 bound to package[0][core:2-3] [node01:103328] MCW rank 3 bound to package[1][core:12-13] The example above shows us that 2 cores have been bound per process. The ":PE=2" qualifier states that 2 CPUs underneath the package (which would be cores in this case) are mapped to each process. $ prun --np 4 --map-by core:PE=2:HWTCPUS --bind-to :REPORT hostname [node01:103506] MCW rank 0 bound to package[0][hwt:0-1] [node01:103506] MCW rank 1 bound to package[0][hwt:8-9] [node01:103506] MCW rank 2 bound to package[0][hwt:16-17] [node01:103506] MCW rank 3 bound to package[0][hwt:24-25] The example above shows us that 2 hardware threads have been bound per process. In this case "prun" is directing the DVM to map by hardware threads since we used the ":HWTCPUS" qualifier. Without that qualifier this command would return an error since by default the DVM will not map to resources smaller than a core. The ":PE=2" qualifier states that 2 processing elements underneath the core (which would be hardware threads in this case) are mapped to each process. $ prun --np 4 --bind-to none:REPORT hostname [node01:107126] MCW rank 0 is not bound (or bound to all available processors) [node01:107126] MCW rank 1 is not bound (or bound to all available processors) [node01:107126] MCW rank 2 is not bound (or bound to all available processors) [node01:107126] MCW rank 3 is not bound (or bound to all available processors) Binding is turned off in the above example, as reported. # [placement-fundamentals] Fundamentals ------------ The mapping of processes to nodes can be defined not just with general policies but also, if necessary, using arbitrary mappings that cannot be described by a simple policy. Supported directives, given on the command line via the "--map-by" option, include: * "SEQ": (often accompanied by the "file=" qualifier) assigns one process to each node specified in the file. The sequential file is to contain an entry for each desired process, one per line of the file. * "RANKFILE": (often accompanied by the "file=" qualifier) assigns one process to the node/resource specified in each entry of the file, one per line of the file. For example, using the hostfile below: $ cat myhostfile aa slots=4 bb slots=4 cc slots=4 The command below will launch three processes, one on each of nodes "aa", "bb", and "cc", respectively. The slot counts don't matter; one process is launched per line on whatever node is listed on the line. $ prun --hostfile myhostfile --map-by seq ./a.out Impact of the ranking option is best illustrated by considering the following hostfile and test cases where each node contains two packages (each package with two cores). Using the "--map-by ppr:2:package" option, we map two processes onto each package and utilize the "--rank-by" option as show below: $ cat myhostfile aa bb +-----------------------------------+-----------------------------------+-----------------------------------+ | Command | Ranks on "aa" | Ranks on "bb" | |===================================|===================================|===================================| | "--rank-by core" | 0 1 ! 2 3 | 4 5 ! 6 7 | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--rank-by package" | 0 2 ! 1 3 | 4 6 ! 5 7 | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--rank-by package:SPAN" | 0 4 ! 1 5 | 2 6 ! 3 7 | +-----------------------------------+-----------------------------------+-----------------------------------+ Ranking by slot provides the identical result as ranking by core in this case — a simple progression of ranks across each node. Ranking by package does a round-robin ranking across packages within each node until all processes have been assigned a rank, and then progresses to the next node. Adding the ":SPAN" qualifier to the ranking directive causes the ranking algorithm to treat the entire allocation as a single entity — thus, the process ranks are assigned across all packages before circling back around to the beginning. The binding operation restricts the process to a subset of the CPU resources on the node. The processors to be used for binding can be identified in terms of topological groupings — e.g., binding to an l3cache will bind each process to all processors within the scope of a single L3 cache within their assigned location. Thus, if a process is assigned by the mapper to a certain package, then a "--bind-to l3cache" directive will cause the process to be bound to the processors that share a single L3 cache within that package. To help balance loads, the binding directive uses a round-robin method, binding a process to the first available specified object type within the object where the process was mapped. For example, consider the case where a job is mapped to the package level, and then bound to core. Each package will have multiple cores, so if multiple processes are mapped to a given package, the binding algorithm will assign each process located to a package to a unique core in a round-robin manner. Binding can only be done to the mapped object or to a resource located within that object. An object is considered completely consumed when the number of processes bound to it equals the number of CPUs within it. Unbound processes are not considered in this computation. Additional processes cannot be mapped to consumed objects unless the OVERLOAD qualifier is provided via the "--bind-to" command line option. Default process mapping/ranking/binding policies can also be set with MCA parameters, overridden by the command line options when provided. MCA parameters can be set on the "prte" command line when starting the DVM (or in the "prterun" command line for a single-execution job), but also in a system or user "mca-params.conf" file or as environment variables, as described in the MCA section below. Some examples include: +-----------------------------------+-----------------------------------+-----------------------------------+ | "prun" option | MCA parameter key | Value | |===================================|===================================|===================================| | "--map-by core" | "rmaps_default_mapping_policy" | "core" | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--map-by package" | "rmaps_default_mapping_policy" | "package" | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--rank-by core" | "rmaps_default_ranking_policy" | "core" | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--bind-to core" | "hwloc_default_binding_policy" | "core`" | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--bind-to package" | "hwloc_default_binding_policy" | "package" | +-----------------------------------+-----------------------------------+-----------------------------------+ | "--bind-to none" | "hwloc_default_binding_policy" | "none" | +-----------------------------------+-----------------------------------+-----------------------------------+ # [placement-limits] Overloading and Oversubscribing ------------------------------- This section explores the difference between the terms "overloading" and "oversubscribing". Users are often confused by the difference between these two scenarios. As such, this section provides a number of scenarios to help illustrate the differences. * "--map-by :OVERSUBSCRIBE" allow more processes on a node than allocated * "--bind-to :overload-allowed" allows for binding more than one process in relation to a CPU The important thing to remember with *oversubscribing* is that it can be defined separately from the actual number of CPUs on a node. This allows the mapper to place more or fewer processes per node than CPUs. By default, PRRTE uses cores to determine slots in the absence of such information provided in the hostfile or by the resource manager (except in the case of the "--host" as described in the section on that command line option. The important thing to remember with *overloading* is that it is defined as binding more processes than CPUs. By default, PRRTE uses cores as a means of counting the number of CPUs. However, the user can adjust this. For example when using the ":HWTCPUS" qualifier to the " --map-by" option PRRTE will use hardware threads as a means of counting the number of CPUs. For the following examples consider a node with: * 2 processor packages, * 10 cores per package, and * 8 hardware threads per core. Consider the node from above with the hostfile below: $ cat myhostfile node01 slots=32 node02 slots=32 The "slots" token tells PRRTE that it can place up to 32 processes before *oversubscribing* the node. If we run the following: prun --np 34 --hostfile myhostfile --map-by core --bind-to core hostname It will return an error at the binding time indicating an *overloading* scenario. The mapping mechanism assigns 32 processes to "node01" matching the "slots" specification in the hostfile. The binding mechanism will bind the first 20 processes to unique cores leaving it with 12 processes that it cannot bind without overloading one of the cores (putting more than one process on the core). Using the "overload-allowed" qualifier to the "--bind-to core" option tells PRRTE that it may assign more than one process to a core. If we run the following: prun --np 34 --hostfile myhostfile --map-by core --bind-to core:overload-allowed hostname This will run correctly placing 32 processes on "node01", and 2 processes on "node02". On "node01" two processes are bound to cores 0-11 accounting for the overloading of those cores. Alternatively, we could use hardware threads to give binding a lower level CPU to bind to without overloading. If we run the following: prun --np 34 --hostfile myhostfile --map-by core:HWTCPUS --bind-to hwthread hostname This will run correctly placing 32 processes on "node01", and 2 processes on "node02". On "node01" two processes are mapped to cores 0-11 but bound to different hardware threads on those cores (the logical first and second hardware thread). Thus no hardware threads are overloaded at binding time. In both of the examples above the node is not oversubscribed at mapping time because the hostfile set the oversubscription limit to "slots=32" for each node. It is only after we exceed that limit that PRRTE will throw an oversubscription error. Consider next if we ran the following: prun --np 66 --hostfile myhostfile --map-by core:HWTCPUS --bind-to hwthread hostname This will return an error at mapping time indicating an oversubscription scenario. The mapping mechanism will assign all of the available slots (64 across 2 nodes) and be left two processes to map. The only way to map those processes is to exceed the number of available slots putting the job into an oversubscription scenario. You can force PRRTE to oversubscribe the nodes by using the ":OVERSUBSCRIBE" qualifier to the "--map-by" option as seen in the example below: prun --np 66 --hostfile myhostfile --map-by core:HWTCPUS:OVERSUBSCRIBE --bind-to hwthread hostname This will run correctly placing 34 processes on "node01" and 32 on "node02". Each process is bound to a unique hardware thread. Overloading vs. Oversubscription: Package Example ================================================= Let's extend these examples by considering the package level. Consider the same node as before, but with the hostfile below: $ cat myhostfile node01 slots=22 node02 slots=22 The lowest level CPUs are "cores" and we have 20 total (10 per package). If we run: prun --np 20 --hostfile myhostfile --map-by package --bind-to package:REPORT hostname Then 10 processes are mapped to each package, and bound at the package level. This is not overloading since we have 10 CPUs (cores) available in the package at the hardware level. However, if we run: prun --np 21 --hostfile myhostfile --map-by package --bind-to package:REPORT hostname Then 11 processes are mapped to the first package and 10 to the second package. At binding time we have an overloading scenario because there are only 10 CPUs (cores) available in the package at the hardware level. So the first package is overloaded. Overloading vs. Oversubscription: Hardware Threads Example ========================================================== Similarly, if we consider hardware threads. Consider the same node as before, but with the hostfile below: $ cat myhostfile node01 slots=165 node02 slots=165 The lowest level CPUs are "hwthreads" (because we are going to use the ":HWTCPUS" qualifier) and we have 160 total (80 per package). If we re-run (from the package example) and add the ":HWTCPUS" qualifier: prun --np 21 --hostfile myhostfile --map-by package:HWTCPUS --bind-to package:REPORT hostname Without the ":HWTCPUS" qualifier this would be overloading (as we saw previously). The mapper places 11 processes on the first package and 10 to the second package. The processes are still bound to the package level. However, with the ":HWTCPUS" qualifier, it is not overloading since we have 80 CPUs (hwthreads) available in the package at the hardware level. Alternatively, if we run: prun --np 161 --hostfile myhostfile --map-by package:HWTCPUS --bind-to package:REPORT hostname Then 81 processes are mapped to the first package and 80 to the second package. At binding time we have an overloading scenario because there are only 80 CPUs (hwthreads) available in the package at the hardware level. So the first package is overloaded. prrte-4.1.1~rc2/src/mca/rmaps/base/base.h0000664000175000017500000001375415203542132020317 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * rmaps framework base functionality. */ #ifndef PRTE_MCA_RMAPS_BASE_H #define PRTE_MCA_RMAPS_BASE_H /* * includes */ #include "prte_config.h" #include "types.h" #include "src/class/pmix_list.h" #include "src/mca/mca.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/mca/schizo/schizo.h" #include "src/util/pmix_printf.h" #include "src/runtime/prte_globals.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS /* * MCA Framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_rmaps_base_framework; /* select a component */ PRTE_EXPORT int prte_rmaps_base_select(void); /* * Global functions for MCA overall collective open and close */ /** * Struct to hold data global to the rmaps framework */ typedef struct { /* list of selected modules */ pmix_list_t selected_modules; /* default mapping/ranking directives */ prte_mapping_policy_t mapping; prte_ranking_policy_t ranking; // default ppr setting char *ppr; /* default device for dist mapping */ char *device; /* whether or not child jobs should inherit mapping/ranking/binding directives from their parent * by default */ bool inherit; /* whether or not we are using hwthreads as independent cpus by default */ bool hwthread_cpus; /* default file for use in sequential and rankfile mapping * when the directive comes thru MCA param */ char *file; // default number of pe's per proc uint16_t default_pes; // some general storage values hwloc_cpuset_t available, baseset; // scratch for binding calculation char *default_mapping_policy; char *default_ranking_policy; /* whether or not to require hwtcpus due to topology limitations */ bool require_hwtcpus; } prte_rmaps_base_t; /** * Global instance of rmaps-wide framework data */ PRTE_EXPORT extern prte_rmaps_base_t prte_rmaps_base; /** * Select an rmaps component / module */ typedef struct { pmix_list_item_t super; int pri; prte_rmaps_base_module_t *module; pmix_mca_base_component_t *component; } prte_rmaps_base_selected_module_t; PMIX_CLASS_DECLARATION(prte_rmaps_base_selected_module_t); /* * Map a job */ PRTE_EXPORT void prte_rmaps_base_map_job(int sd, short args, void *cbdata); /** * Utility routines to get/set vpid mapping for the job */ PRTE_EXPORT int prte_rmaps_base_get_vpid_range(pmix_nspace_t jobid, pmix_rank_t *start, pmix_rank_t *range); PRTE_EXPORT int prte_rmaps_base_set_vpid_range(pmix_nspace_t jobid, pmix_rank_t start, pmix_rank_t range); /* pretty-print functions */ PRTE_EXPORT char *prte_rmaps_base_print_mapping(prte_mapping_policy_t mapping); PRTE_EXPORT char *prte_rmaps_base_print_ranking(prte_ranking_policy_t ranking); PRTE_EXPORT int prte_rmaps_base_prep_topology(hwloc_topology_t topo); PRTE_EXPORT int prte_rmaps_base_filter_nodes(prte_app_context_t *app, pmix_list_t *nodes, bool remove); PRTE_EXPORT int prte_rmaps_base_set_default_mapping(prte_job_t *jdata, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_set_mapping_policy(prte_job_t *jdata, char *spec); PRTE_EXPORT int prte_rmaps_base_set_default_ranking(prte_job_t *jdata, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_set_ranking_policy(prte_job_t *jdata, char *spec); PRTE_EXPORT void prte_rmaps_base_display_map(prte_job_t *jdata); PRTE_EXPORT void prte_rmaps_base_report_bindings(prte_job_t *jdata, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_get_ncpus(prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options); PRTE_EXPORT bool prte_rmaps_base_check_avail(prte_job_t *jdata, prte_app_context_t *app, prte_node_t *node, pmix_list_t *node_list, hwloc_obj_t obj, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_check_oversubscribed(prte_job_t *jdata, prte_app_context_t *app, prte_node_t *node, prte_rmaps_options_t *options); PRTE_EXPORT void prte_rmaps_base_get_cpuset(prte_job_t *jdata, prte_node_t *node, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_check_support(prte_job_t *jdata, prte_node_t *node, prte_rmaps_options_t *options); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_binding.c0000664000175000017500000004272615203542132023207 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018 Inria. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/threads/pmix_tsd.h" #include "src/util/pmix_if.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/runtime/prte_globals.h" #include "src/util/dash_host/dash_host.h" #include "src/util/hostfile/hostfile.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "types.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" static int bind_generic(prte_job_t *jdata, prte_proc_t *proc, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options) { hwloc_obj_t trg_obj = NULL, tmp_obj = NULL; unsigned ncpus; hwloc_obj_type_t type; hwloc_obj_t target; hwloc_cpuset_t tgtcpus, tmpcpus; int nobjs, n; prte_hwloc_obj_data_t *objcnt; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: bind %s to %s with policy %s", PRTE_NAME_PRINT(&proc->name), hwloc_obj_type_string(options->maptype), prte_hwloc_base_print_binding(jdata->map->binding)); /* initialize */ if (NULL == obj) { target = hwloc_get_root_obj(node->topology->topo); } else { target = obj; } if (NULL == options->target) { return PRTE_ERROR; } tgtcpus = target->cpuset; hwloc_bitmap_and(prte_rmaps_base.baseset, options->target, tgtcpus); nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->hwb); // check for target object existence if (0 == nobjs) { // if this is not a default binding policy, then error out if (PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { pmix_show_help("help-prte-rmaps-base.txt", "rmaps:binding-target-not-found", true, prte_hwloc_base_print_binding(jdata->map->binding), node->name); return PRTE_ERR_SILENT; } // fallback to not binding return PRTE_SUCCESS; } for (n=0; n < nobjs; n++) { tmp_obj = prte_hwloc_base_get_obj_by_type(node->topology->topo, options->hwb, n); // if a limit on the number of procs/object has been set, // then check it here if (NULL == tmp_obj->userdata) { objcnt = PMIX_NEW(prte_hwloc_obj_data_t); tmp_obj->userdata = (void*)objcnt; } else { objcnt = (prte_hwloc_obj_data_t*)tmp_obj->userdata; } if (0 < options->limit && options->limit <= objcnt->nprocs) { // skip this object continue; } tmpcpus = tmp_obj->cpuset; hwloc_bitmap_and(prte_rmaps_base.available, node->available, tmpcpus); hwloc_bitmap_and(prte_rmaps_base.available, prte_rmaps_base.available, prte_rmaps_base.baseset); if (options->use_hwthreads) { ncpus = hwloc_bitmap_weight(prte_rmaps_base.available); } else { /* if we are treating cores as cpus, then we really * want to know how many cores are in this object. * hwloc sets a bit for each "pu", so we can't just * count bits in this case as there may be more than * one hwthread/core. Instead, find the number of cores * under the object */ ncpus = hwloc_get_nbobjs_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, HWLOC_OBJ_CORE); } if (0 < ncpus) { trg_obj = tmp_obj; if (0 < options->limit) { objcnt->nprocs++; } break; } } if (NULL == trg_obj) { /* there aren't any appropriate targets under this object */ if (PRTE_BINDING_REQUIRED(jdata->map->binding)) { pmix_show_help("help-prte-rmaps-base.txt", "rmaps:no-available-cpus", true, node->name); return PRTE_ERR_SILENT; } else { return PRTE_SUCCESS; } } tgtcpus = trg_obj->cpuset; if (NULL == tgtcpus) { return PRTE_ERROR; } hwloc_bitmap_list_asprintf(&proc->cpuset, tgtcpus); // bind to the entire target object if (4 < pmix_output_get_verbosity(prte_rmaps_base_framework.framework_output)) { char *tmp1; bool physical; physical = prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_PHYSICAL_CPUS, NULL, PMIX_BOOL); tmp1 = prte_hwloc_base_cset2str(trg_obj->cpuset, options->use_hwthreads, physical, node->topology->topo); pmix_output(prte_rmaps_base_framework.framework_output, "%s BOUND PROC %s[%s] TO %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proc->name), node->name, tmp1); free(tmp1); } /* mark the assigned cpus as having been consumed. If we are binding to * an object that has multiple CPUs, then we only mark one here to indicate * that a process was bound to that object. This provides an accounting * mechanism that lets us know when we become overloaded - i.e., more procs * are bound to the object than there are CPUs. We must also mark the bits * in the options->target cpuset so that the next proc we assign doesn't * attempt to take the same location. */ if (options->use_hwthreads) { type = HWLOC_OBJ_PU; } else { type = HWLOC_OBJ_CORE; } tmp_obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, type, 0); if (NULL == tmp_obj) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); if (PRTE_BINDING_REQUIRED(jdata->map->binding)) { pmix_show_help("help-prte-rmaps-base.txt", "rmaps:no-available-cpus", true, node->name); return PRTE_ERR_SILENT; } else { return PRTE_SUCCESS; } } hwloc_bitmap_andnot(node->available, node->available, tmp_obj->cpuset); if (hwloc_bitmap_iszero(node->available) && options->overload) { /* reset the availability */ hwloc_bitmap_copy(node->available, node->jobcache); } return PRTE_SUCCESS; } static int bind_to_cpuset(prte_job_t *jdata, prte_proc_t *proc, prte_node_t *node, prte_rmaps_options_t *options) { /* bind each process to prte_hwloc_base_cpu_list */ unsigned idx; hwloc_bitmap_t tset; hwloc_obj_t obj, root, pkg; char **cpus; hwloc_obj_type_t type; unsigned n, npkgs; int rc; bool included; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: bind job %s to cpus %s %s", PRTE_JOBID_PRINT(jdata->nspace), options->cpuset, options->ordered ? "ordered" : "not-ordered"); if (NULL == options->cpuset) { /* not enough cpus were specified */ return PRTE_ERR_OUT_OF_RESOURCE; } cpus = PMIx_Argv_split(options->cpuset, ','); /* take the first one */ idx = strtoul(cpus[0], NULL, 10); if (options->use_hwthreads) { type = HWLOC_OBJ_PU; } else { type = HWLOC_OBJ_CORE; } /* the CPU numbers would have been given to us based on the total * available CPUs on the machine. Thus, we cannot use the node->available * CPU set as we are removing CPUs for accounting purposes there. * Instead, refer back to the root-level information */ root = hwloc_get_root_obj(node->topology->topo); if (options->ordered) { /* assign each proc to the next available * cpu in the list. Since we are assigning * procs as they are mapped, this ensures they * will be assigned in order */ tset = root->cpuset; obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, tset, type, idx); if (NULL == obj) { PMIx_Argv_free(cpus); return PRTE_ERR_OUT_OF_RESOURCE; } tset = obj->cpuset; } else { /* bind the proc to all assigned cpus */ tset = options->target; } /* sanity check - are all the target cpus in a single * package, or do they span packages? */ npkgs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE); included = false; for (n=0; n < npkgs; n++) { pkg = prte_hwloc_base_get_obj_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE, n); rc = hwloc_bitmap_isincluded(tset, pkg->cpuset); if (1 == rc) { included = true; break; } } if (!included) { pmix_show_help("help-prte-rmaps-base.txt", "span-packages-cpuset", true, prte_rmaps_base_print_mapping(jdata->map->mapping), prte_hwloc_base_print_binding(jdata->map->binding), options->cpuset); PMIx_Argv_free(cpus); return PRTE_ERR_SILENT; } /* bind to the specified cpuset */ hwloc_bitmap_list_asprintf(&proc->cpuset, tset); /* remove one of the CPUs from the cpuset to indicate that * we assigned a proc to this range */ free(options->cpuset); if (NULL == cpus[1]) { options->cpuset = NULL; } else { options->cpuset = PMIx_Argv_join(&cpus[1], ','); } PMIx_Argv_free(cpus); /* mark that we used ONE of these cpus - we do this each time we * the cpuset is assigned to a proc. When all the cpus in the * set have been removed, we know that the set will be overloaded * if any more procs are assigned to it. */ tset = root->cpuset; obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, tset, type, idx); if (NULL != obj) { hwloc_bitmap_andnot(node->available, node->available, obj->cpuset); } return PRTE_SUCCESS; } static int bind_multiple(prte_job_t *jdata, prte_proc_t *proc, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options) { hwloc_obj_type_t type; hwloc_cpuset_t result, tgtcpus; hwloc_obj_t target, tmp_obj, pkg; uint16_t n; unsigned npkgs, ncpus; bool moveon = false; PRTE_HIDE_UNUSED_PARAMS(jdata); pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: bind proc %s to %d cpus", PRTE_NAME_PRINT(&proc->name), options->cpus_per_rank); /* initialize */ result = hwloc_bitmap_alloc(); hwloc_bitmap_zero(result); if (NULL == obj) { target = hwloc_get_root_obj(node->topology->topo); } else { target = obj; } tgtcpus = target->cpuset; hwloc_bitmap_and(prte_rmaps_base.baseset, options->target, tgtcpus); if (options->use_hwthreads) { type = HWLOC_OBJ_PU; } else { type = HWLOC_OBJ_CORE; } if (NULL == obj) { /* we do not want the cpu envelope to split across * packages, so we need to ensure we set the * available processors to cover whichever package * has enough CPUs to fill the request */ npkgs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE); for (n=0; n < npkgs; n++) { pkg = prte_hwloc_base_get_obj_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE, n); hwloc_bitmap_and(prte_rmaps_base.available, prte_rmaps_base.baseset, pkg->cpuset); hwloc_bitmap_and(prte_rmaps_base.available, prte_rmaps_base.available, node->available); ncpus = hwloc_get_nbobjs_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, type); if (ncpus >= options->cpus_per_rank) { /* this is a good spot */ moveon = true; break; } } if (!moveon) { /* if we get here, then there are no packages that can completely * cover the request - so return an error */ hwloc_bitmap_free(result); pmix_show_help("help-prte-rmaps-base.txt", "span-packages-multiple", true, prte_rmaps_base_print_mapping(jdata->map->mapping), prte_hwloc_base_print_binding(jdata->map->binding), options->cpus_per_rank, node->name); return PRTE_ERR_SILENT; } } else { hwloc_bitmap_and(prte_rmaps_base.available, prte_rmaps_base.baseset, node->available); } /* we bind-to-cpu for the number of cpus that was specified, * restricting ourselves to the available cpus in the object */ for (n=0; n < options->cpus_per_rank; n++) { tmp_obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, type, n); if (NULL != tmp_obj) { hwloc_bitmap_or(result, result, tmp_obj->cpuset); hwloc_bitmap_andnot(node->available, node->available, tmp_obj->cpuset); hwloc_bitmap_andnot(options->target, options->target, tmp_obj->cpuset); } } hwloc_bitmap_list_asprintf(&proc->cpuset, result); hwloc_bitmap_free(result); if (NULL == proc->cpuset || 0 == strlen(proc->cpuset)) { pmix_show_help("help-prte-rmaps-base.txt", "not-enough-cpus", true, options->pprn, hwloc_obj_type_string(options->maptype), options->cpus_per_rank); return PRTE_ERR_SILENT; } return PRTE_SUCCESS; } int prte_rmaps_base_bind_proc(prte_job_t *jdata, prte_proc_t *proc, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options) { int rc; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps: compute bindings for job %s with policy %s[%x]", PRTE_JOBID_PRINT(jdata->nspace), prte_hwloc_base_print_binding(jdata->map->binding), jdata->map->binding); if (PRTE_MAPPING_BYUSER == options->map) { /* user specified binding by rankfile or it was computed * by the likwid mapper - nothing for us to do */ return PRTE_SUCCESS; } if (PRTE_BIND_TO_NONE == options->bind) { rc = PRTE_SUCCESS; if (NULL != options->cpuset && !(PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL))) { /* "soft" cgroup was given but no other * binding directive was provided, so bind * to those specific cpus */ if (PRTE_SUCCESS != (rc = bind_to_cpuset(jdata, proc, node, options))) { PRTE_ERROR_LOG(rc); } } return rc; } /* some systems do not report cores, and so we can get a situation where our * default binding policy will fail for no necessary reason. So if we are * computing a binding due to our default policy, and no cores are found * on this node, just silently skip it - we will not bind */ if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding) && HWLOC_TYPE_DEPTH_UNKNOWN == hwloc_get_type_depth(node->topology->topo, HWLOC_OBJ_CORE)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "Unable to bind-to core by default on node %s as no cores detected", node->name); return PRTE_SUCCESS; } if (PRTE_MAPPING_PELIST == options->map) { rc = bind_to_cpuset(jdata, proc, node, options); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } return rc; } if (1 < options->cpus_per_rank) { rc = bind_multiple(jdata, proc, node, obj, options); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } return rc; } rc = bind_generic(jdata, proc, node, obj, options); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } return rc; } prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_support_fns.c0000664000175000017500000010720115203542132024145 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016-2021 IBM Corporation. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/threads/pmix_tsd.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_if.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/runtime/prte_globals.h" #include "src/util/dash_host/dash_host.h" #include "src/util/hostfile/hostfile.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "types.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" int prte_rmaps_base_filter_nodes(prte_app_context_t *app, pmix_list_t *nodes, bool remove) { int rc = PRTE_ERR_TAKE_NEXT_OPTION; char *hosts; /* did the app_context contain a hostfile? */ hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_HOSTFILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { /* yes - filter the node list through the file, removing * any nodes not found in the file */ if (PRTE_SUCCESS != (rc = prte_util_filter_hostfile_nodes(nodes, hosts, remove))) { PRTE_ERROR_LOG(rc); free(hosts); return rc; } /** check that anything is here */ if (0 == pmix_list_get_size(nodes)) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-mapped-node", true, app->app, "-hostfile", hosts); free(hosts); return PRTE_ERR_SILENT; } free(hosts); } /* now filter the list through any -host specification */ hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { if (PRTE_SUCCESS != (rc = prte_util_filter_dash_host_nodes(nodes, hosts, remove))) { PRTE_ERROR_LOG(rc); free(hosts); return rc; } /** check that anything is left! */ if (0 == pmix_list_get_size(nodes)) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-mapped-node", true, app->app, "-host", hosts); free(hosts); return PRTE_ERR_SILENT; } free(hosts); } return rc; } /* * Query the registry for all nodes allocated to a specified app_context */ int prte_rmaps_base_get_target_nodes(pmix_list_t *allocated_nodes, int32_t *total_num_slots, prte_job_t *jdata, prte_app_context_t *app, prte_mapping_policy_t policy, bool initial_map, bool silent, bool keepall) { pmix_list_item_t *item; prte_node_t *node, *nd, *nptr, *next; int32_t num_slots; int32_t i; int rc; prte_job_t *daemons; bool novm; pmix_list_t nodes; char *hosts = NULL; bool needhosts = false; /** set default answer */ *total_num_slots = 0; /* get the daemon job object */ daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* see if we have a vm or not */ novm = prte_get_attribute(&daemons->attributes, PRTE_JOB_NO_VM, NULL, PMIX_BOOL); /* if this is NOT a managed allocation, then we use the nodes * that were specified for this app - there is no need to collect * all available nodes and "filter" them. * * However, if it is a managed allocation AND the hostfile or the hostlist was * provided, those take precedence, so process them and filter as we normally do. */ hosts = NULL; if ((prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING) || prte_get_attribute(&app->attributes, PRTE_APP_HOSTFILE, (void **) &hosts, PMIX_STRING)) && NULL != hosts) { needhosts = true; free(hosts); } if (!prte_managed_allocation || (prte_managed_allocation && needhosts)) { PMIX_CONSTRUCT(&nodes, pmix_list_t); /* if the app provided a dash-host, then use those nodes */ hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s using dash_host %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); if (PRTE_SUCCESS != (rc = prte_util_add_dash_host_nodes(&nodes, hosts, false))) { PRTE_ERROR_LOG(rc); free(hosts); return rc; } free(hosts); } else if (prte_get_attribute(&app->attributes, PRTE_APP_HOSTFILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { /* otherwise, if the app provided a hostfile, then use that */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s using hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); if (PRTE_SUCCESS != (rc = prte_util_add_hostfile_nodes(&nodes, hosts))) { free(hosts); PRTE_ERROR_LOG(rc); return rc; } free(hosts); } else { /* if nothing else was specified by the app, then use all known nodes, which * will include ourselves */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s using known nodes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto addknown; } /** if we still don't have anything */ if (0 == pmix_list_get_size(&nodes)) { if (!silent) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-available-resources", true); } PMIX_DESTRUCT(&nodes); return PRTE_ERR_SILENT; } /* find the nodes in the session and assemble them * in list order as that is what the user specified. Note * that the prte_node_t objects on the nodes list are not * fully filled in - they only contain the user-provided * name of the node as a temp object. Thus, we cannot just * check to see if the node pointer matches that of a node * in the session. */ PMIX_LIST_FOREACH_SAFE(nptr, next, &nodes, prte_node_t) { for (i = 0; i < jdata->session->nodes->size; i++) { node = (prte_node_t *) pmix_pointer_array_get_item(jdata->session->nodes, i); if (NULL == node) { continue; } /* ignore nodes that are non-usable */ if (PRTE_FLAG_TEST(node, PRTE_NODE_NON_USABLE)) { continue; } /* ignore nodes that are marked as do-not-use for this mapping */ if (PRTE_NODE_STATE_DO_NOT_USE == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS MARKED NO_USE", node->name)); /* reset the state so it can be used another time */ node->state = PRTE_NODE_STATE_UP; continue; } if (PRTE_NODE_STATE_DOWN == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS DOWN", node->name)); continue; } if (PRTE_NODE_STATE_NOT_INCLUDED == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS MARKED NO_INCLUDE", node->name)); /* not to be used */ continue; } /* if this node wasn't included in the vm (e.g., by -host), ignore it, * unless we are mapping prior to launching the vm */ if (NULL == node->daemon && !novm) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s HAS NO DAEMON", node->name)); continue; } if (!prte_nptr_match(node, nptr)) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s DOESNT MATCH NODE %s", node->name, nptr->name)); continue; } /* retain a copy for our use in case the item gets * destructed along the way */ PMIX_RETAIN(node); if (initial_map) { /* if this is the first app_context we * are getting for an initial map of a job, * then mark all nodes as unmapped */ PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } /* the list is ordered as per user direction using -host * or the listing in -hostfile - preserve that ordering */ pmix_list_append(allocated_nodes, &node->super); break; } /* remove the item from the list as we have allocated it */ pmix_list_remove_item(&nodes, (pmix_list_item_t *) nptr); PMIX_RELEASE(nptr); } PMIX_DESTRUCT(&nodes); /* now prune for usage and compute total slots */ goto complete; } addknown: /* add everything in the node pool that can be used - add them * in daemon order, which may be different than the order in the * node pool. Since an empty list is passed into us, the list at * this point either has the HNP node or nothing, and the HNP * node obviously has a daemon on it (us!) */ if (0 == pmix_list_get_size(allocated_nodes)) { nd = NULL; } else { nd = (prte_node_t *) pmix_list_get_last(allocated_nodes); } for (i = 0; i < jdata->session->nodes->size; i++) { if (NULL != (node = (prte_node_t *) pmix_pointer_array_get_item(jdata->session->nodes, i))) { /* ignore nodes that are non-usable */ if (PRTE_FLAG_TEST(node, PRTE_NODE_NON_USABLE)) { continue; } /* ignore nodes that are marked as do-not-use for this mapping */ if (PRTE_NODE_STATE_DO_NOT_USE == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS MARKED NO_USE", node->name)); /* reset the state so it can be used another time */ node->state = PRTE_NODE_STATE_UP; continue; } if (PRTE_NODE_STATE_DOWN == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS MARKED DOWN", node->name)); continue; } if (PRTE_NODE_STATE_NOT_INCLUDED == node->state) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s IS MARKED NO_INCLUDE", node->name)); /* not to be used */ continue; } /* if this node wasn't included in the vm (e.g., by -host), ignore it, * unless we are mapping prior to launching the vm */ if (NULL == node->daemon && !novm) { PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "NODE %s HAS NO DAEMON", node->name)); continue; } if(0 == node->index && !prte_hnp_is_allocated){ PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "HNP NODE %s IS NOT ALLOCATED", node->name)); continue; } /* retain a copy for our use in case the item gets * destructed along the way */ PMIX_RETAIN(node); if (initial_map) { /* if this is the first app_context we * are getting for an initial map of a job, * then mark all nodes as unmapped */ PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } if (NULL == nd || NULL == nd->daemon || NULL == node->daemon || nd->daemon->name.rank < node->daemon->name.rank) { /* just append to end */ pmix_list_append(allocated_nodes, &node->super); nd = node; } else { /* starting from end, put this node in daemon-vpid order */ while (node->daemon->name.rank < nd->daemon->name.rank) { if (pmix_list_get_begin(allocated_nodes) == pmix_list_get_prev(&nd->super)) { /* insert at beginning */ pmix_list_prepend(allocated_nodes, &node->super); goto moveon; } nd = (prte_node_t *) pmix_list_get_prev(&nd->super); } item = pmix_list_get_next(&nd->super); if (item == pmix_list_get_end(allocated_nodes)) { /* we are at the end - just append */ pmix_list_append(allocated_nodes, &node->super); } else { pmix_list_insert_pos(allocated_nodes, item, &node->super); } moveon: /* reset us back to the end for the next node */ nd = (prte_node_t *) pmix_list_get_last(allocated_nodes); } } } PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Starting with %d nodes in list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) pmix_list_get_size(allocated_nodes))); /** check that anything is here */ if (0 == pmix_list_get_size(allocated_nodes)) { if (!silent) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:no-available-resources", true); } return PRTE_ERR_SILENT; } /* filter the nodes thru any hostfile and dash-host options */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Filtering thru apps", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (PRTE_SUCCESS != (rc = prte_rmaps_base_filter_nodes(app, allocated_nodes, true)) && PRTE_ERR_TAKE_NEXT_OPTION != rc) { PRTE_ERROR_LOG(rc); return rc; } PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Retained %d nodes in list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) pmix_list_get_size(allocated_nodes))); complete: /* if we are mapping debuggers, then they don't count against * the allocation */ if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { num_slots = INT32_MAX; PMIX_LIST_FOREACH_SAFE(node, next, allocated_nodes, prte_node_t) { if (0 == node->index) { if (!prte_hnp_is_allocated || (PRTE_GET_MAPPING_DIRECTIVE(policy) & PRTE_MAPPING_NO_USE_LOCAL)) { pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); /* "un-retain" it */ continue; } } if (NULL == node->topology || NULL == node->topology->topo) { /* cannot use this node - should never happen */ pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); continue; } /* cache the available CPUs for later */ hwloc_bitmap_copy(node->jobcache, node->available); } } else { num_slots = 0; PMIX_LIST_FOREACH_SAFE(node, next, allocated_nodes, prte_node_t) { if (NULL == node->topology || NULL == node->topology->topo) { PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s node %s lacks topology", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); /* cannot use this node - should never happen */ pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); continue; } /* if the hnp was not allocated, or flagged not to be used, * then remove it here */ if (!prte_hnp_is_allocated || (PRTE_GET_MAPPING_DIRECTIVE(policy) & PRTE_MAPPING_NO_USE_LOCAL)) { if (0 == node->index) { PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s node %s HNP not allocated or not to be used", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); /* "un-retain" it */ continue; } } /** check to see if this node is fully used - remove if so */ if (!keepall && 0 != node->slots_max && node->slots_inuse >= node->slots_max) { PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Removing node %s: max %d inuse %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, node->slots_max, node->slots_inuse)); pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); /* "un-retain" it */ continue; } if (!keepall && node->slots <= node->slots_inuse && (PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(policy))) { /* remove the node as fully used */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Removing node %s slots %d inuse %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, node->slots, node->slots_inuse)); pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); /* "un-retain" it */ continue; } if (node->slots > node->slots_inuse) { int32_t s; /* check for any -host allocations */ hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { s = prte_util_dash_host_compute_slots(node, hosts); free(hosts); } else { s = node->slots - node->slots_inuse; } node->slots_available = s; /* add the available slots */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s node %s has %d slots available", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, s)); num_slots += s; /* cache the available CPUs for later */ hwloc_bitmap_copy(node->jobcache, node->available); continue; } if (!(PRTE_MAPPING_NO_OVERSUBSCRIBE & PRTE_GET_MAPPING_DIRECTIVE(policy))) { /* nothing needed to do here - we don't add slots to the * count as we don't have any available. Just let the mapper * do what it needs to do to meet the request */ PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s node %s is fully used, but available for oversubscription", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); /* cache the available CPUs for later */ hwloc_bitmap_copy(node->jobcache, node->available); } else if (!keepall) { PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s node %s is fully used and not available for oversubscription: SLOTS %d INUSE %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, (int)node->slots, (int)node->slots_inuse)); /* if we cannot use it, remove it from list */ pmix_list_remove_item(allocated_nodes, &node->super); PMIX_RELEASE(node); /* "un-retain" it */ continue; } } } /* Sanity check to make sure we have resources available */ if (0 == pmix_list_get_size(allocated_nodes)) { if (silent) { /* let the caller know that the resources exist, * but are currently busy */ return PRTE_ERR_RESOURCE_BUSY; } else { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:all-available-resources-used", true); return PRTE_ERR_SILENT; } } /* pass back the total number of available slots */ *total_num_slots = num_slots; /* check for prior bookmark */ prte_rmaps_base_get_starting_point(allocated_nodes, jdata); if (4 < pmix_output_get_verbosity(prte_rmaps_base_framework.framework_output)) { pmix_output(0, "AVAILABLE NODES FOR MAPPING:"); for (item = pmix_list_get_first(allocated_nodes); item != pmix_list_get_end(allocated_nodes); item = pmix_list_get_next(item)) { node = (prte_node_t *) item; pmix_output(0, " node: %s daemon: %s slots_available: %d", node->name, (NULL == node->daemon) ? "NULL" : PRTE_VPID_PRINT(node->daemon->name.rank), node->slots_available); } } return PRTE_SUCCESS; } prte_proc_t *prte_rmaps_base_setup_proc(prte_job_t *jdata, prte_app_idx_t idx, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options) { prte_proc_t *proc; int rc; prte_app_context_t *app; proc = PMIX_NEW(prte_proc_t); /* set the jobid */ PMIX_LOAD_NSPACE(proc->name.nspace, jdata->nspace); /* flag the proc as ready for launch */ proc->state = PRTE_PROC_STATE_INIT; proc->app_idx = idx; app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, idx); if (NULL == app) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PMIX_RELEASE(proc); return NULL; } /* mark the proc as UPDATED so it will be included in the launch */ PRTE_FLAG_SET(proc, PRTE_PROC_FLAG_UPDATED); if (NULL == node->daemon) { proc->parent = PMIX_RANK_INVALID; } else { proc->parent = node->daemon->name.rank; } // point the proc at its node proc->node = node; PMIX_RETAIN(node); /* maintain accounting on object */ /* point the proc to its locale */ proc->obj = obj; /* bind the process so we know which cpus have been taken */ rc = prte_rmaps_base_bind_proc(jdata, proc, node, obj, options); if (PRTE_SUCCESS != rc) { PMIX_RELEASE(proc); // releases node to maintain accounting return NULL; } if (0 > (rc = pmix_pointer_array_add(node->procs, (void *) proc))) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(proc); // releases node to maintain accounting return NULL; } /* if this is a tool app, then it doesn't count against * available slots - otherwise, it does */ if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { proc->local_rank = 0; proc->node_rank = UINT16_MAX; } else { proc->node_rank = node->num_procs; node->num_procs++; ++node->slots_inuse; } /* retain the proc struct so that we correctly track its release */ PMIX_RETAIN(proc); return proc; } /* * determine the proper starting point for the next mapping operation */ void prte_rmaps_base_get_starting_point(pmix_list_t *node_list, prte_job_t *jdata) { prte_node_t *node, *nd1; bool first = true; /* if a bookmark exists from some prior mapping, set us to start there */ node = NULL; if (NULL != jdata->bookmark) { /* find this node on the list */ PMIX_LIST_FOREACH(nd1, node_list, prte_node_t) { if (nd1->index == jdata->bookmark->index) { node = nd1; break; } first = false; } } if (NULL == node || first) { return; } PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s Starting bookmark at node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); /* put this node at the front of the list */ pmix_list_remove_item(node_list, &node->super); pmix_list_prepend(node_list, &node->super); return; } int prte_rmaps_base_get_ncpus(prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options) { int ncpus; if (NULL == options->job_cpuset) { hwloc_bitmap_copy(prte_rmaps_base.available, node->available); } else { hwloc_bitmap_and(prte_rmaps_base.available, node->available, options->job_cpuset); } if (NULL != obj) { hwloc_bitmap_and(prte_rmaps_base.available, prte_rmaps_base.available, obj->cpuset); } if (options->use_hwthreads) { ncpus = hwloc_bitmap_weight(prte_rmaps_base.available); } else { /* if we are treating cores as cpus, then we really * want to know how many cores are in this object. * hwloc sets a bit for each "pu", so we can't just * count bits in this case as there may be more than * one hwthread/core. Instead, find the number of cores * under the object */ ncpus = hwloc_get_nbobjs_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, HWLOC_OBJ_CORE); } return ncpus; } bool prte_rmaps_base_check_avail(prte_job_t *jdata, prte_app_context_t *app, prte_node_t *node, pmix_list_t *node_list, hwloc_obj_t obj, prte_rmaps_options_t *options) { int nprocs; bool avail = false; pmix_output_verbose(10, prte_rmaps_base_framework.framework_output, "%s get_avail_ncpus: node %s has %d procs on it", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, node->num_procs); if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { avail = true; goto done; } if (!options->oversubscribe) { if (node->slots <= node->slots_inuse) { pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps: node %s is full - skipping", node->name); goto done; } } if (0 != node->slots_max && node->slots_max <= node->slots_inuse) { /* cannot use this node - already at max_slots */ pmix_list_remove_item(node_list, &node->super); PMIX_RELEASE(node); goto done; } if (PRTE_BIND_TO_NONE == options->bind) { if (NULL != options->job_cpuset) { options->target = hwloc_bitmap_dup(options->job_cpuset); } else { options->target = NULL; } avail = true; goto done; } options->ncpus = prte_rmaps_base_get_ncpus(node, obj, options); /* the available cpus are in the scratch location */ options->target = hwloc_bitmap_dup(prte_rmaps_base.available); // compute how many procs we can support on this object nprocs = options->ncpus / options->cpus_per_rank; if (0 < options->pprn) { // we are ppr mapping, so there must be enough cpus to // support at least pprn procs on this object if (options->pprn <= nprocs) { avail = true; } } else { if (options->nprocs <= nprocs) { avail = true; } else if (options->overload) { /* doesn't matter how many cpus are in use */ avail = true; } else if (0 < nprocs) { options->nprocs = nprocs; avail = true; } } done: /* add this node to the map - do it only once */ if (avail && !PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_MAPPED)) { PRTE_FLAG_SET(node, PRTE_NODE_FLAG_MAPPED); PMIX_RETAIN(node); pmix_pointer_array_add(jdata->map->nodes, node); ++(jdata->map->num_nodes); options->nnodes++; // track #nodes for this app } return avail; } void prte_rmaps_base_get_cpuset(prte_job_t *jdata, prte_node_t *node, prte_rmaps_options_t *options) { PRTE_HIDE_UNUSED_PARAMS(jdata); if (NULL != options->cpuset) { options->job_cpuset = prte_hwloc_base_generate_cpuset(node->topology->topo, options->use_hwthreads, &options->cpuset); } else { options->job_cpuset = hwloc_bitmap_dup(node->available); } } int prte_rmaps_base_check_support(prte_job_t *jdata, prte_node_t *node, prte_rmaps_options_t *options) { struct hwloc_topology_support *support; if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL) || PRTE_BIND_TO_NONE == PRTE_GET_BINDING_POLICY(jdata->map->binding)) { return PRTE_SUCCESS; } support = (struct hwloc_topology_support *) hwloc_topology_get_support(node->topology->topo); /* check if topology supports cpubind - have to be careful here * as Linux doesn't currently support thread-level binding. This * may change in the future, though, and it isn't clear how hwloc * interprets the current behavior. So check both flags to be sure. */ if (!support->cpubind->set_thisproc_cpubind && !support->cpubind->set_thisthread_cpubind) { if (PRTE_BINDING_REQUIRED(jdata->map->binding) && PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { /* we are required to bind but cannot */ pmix_show_help("help-prte-rmaps-base.txt", "rmaps:cpubind-not-supported", true, node->name); return PRTE_ERR_SILENT; } } /* check if topology supports membind - have to be careful here * as hwloc treats this differently than I (at least) would have * expected. Per hwloc, Linux memory binding is at the thread, * and not process, level. Thus, hwloc sets the "thisproc" flag * to "false" on all Linux systems, and uses the "thisthread" flag * to indicate binding capability - don't warn if the user didn't * specifically request binding */ if (!support->membind->set_thisproc_membind && !support->membind->set_thisthread_membind && PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { if (PRTE_HWLOC_BASE_MBFA_WARN == prte_hwloc_base_mbfa && !options->membind_warned) { pmix_show_help("help-prte-rmaps-base.txt", "rmaps:membind-not-supported", true, node->name); options->membind_warned = true; } else if (PRTE_HWLOC_BASE_MBFA_ERROR == prte_hwloc_base_mbfa) { pmix_show_help("help-prte-rmaps-base.txt", "rmaps:membind-not-supported-fatal", true, node->name); return PRTE_ERR_SILENT; } } return PRTE_SUCCESS; } int prte_rmaps_base_check_oversubscribed(prte_job_t *jdata, prte_app_context_t *app, prte_node_t *node, prte_rmaps_options_t *options) { if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { return PRTE_SUCCESS; } if (!options->oversubscribe && node->slots == node->num_procs) { return PRTE_ERR_TAKE_NEXT_OPTION; // this node is full } /* not all nodes are equal, so only set oversubscribed for * this node if it is in that state */ if (node->slots < (int) node->num_procs) { /* flag the node as oversubscribed so that sched-yield gets * properly set */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_OVERSUBSCRIBED); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_OVERSUBSCRIBED); if (options->oversubscribe) { return PRTE_SUCCESS; } /* check for permission */ if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { /* if we weren't given a directive either way, then we will error out * as the #slots were specifically given, either by the host RM or * via hostfile/dash-host */ if (!(PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping))) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } else if (!options->oversubscribe) { /* if we were explicitly told not to oversubscribe, then don't */ pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } } } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/base/help-bindto.txt0000664000175000017500000001003615203542132022170 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRRTE's prun. # [bind-to] By default, processes are bound to individual CPUs (either COREs or HWTHREADs, as defined by default or by user specification for the job). On nodes that are OVERSUBSCRIBEd (i.e., where the number of procs exceeds the number of assigned slots), the default is to not bind the processes. Note: Processes from prior jobs that are already executing on a node are not "unbound" when a new job mapping results in the node becoming oversubscribed. Binding is performed to the first available specified object type within the object where the process was mapped. In other words, binding can only be done to the mapped object or to a resource located beneath that object. An object is considered completely consumed when the number of processes bound to it equals the number of CPUs within it. Unbound processes are not considered in this computation. Additional processes cannot be mapped to consumed objects unless the "OVERLOAD" qualifier is provided via the "--bind-to" command line option. Note that directives and qualifiers are case-insensitive and can be shortened to the minimum number of characters to uniquely identify them. Thus, "L1CACHE" can be given as "l1cache" or simply as "L1". Supported binding directives include: * "NONE" does not bind the processes * "HWTHREAD" binds each process to a single hardware thread/ This requires that hwthreads be treated as independent CPUs (i.e., that either the "HWTCPUS" qualifier be provided to the "map-by" option or that "hwthreads" be designated as CPUs by default). * "CORE" binds each process to a single core. This can be done whether "hwthreads" or "cores" are being treated as independent CPUs provided that mapping is performed at the core or higher level. * "L1CACHE" binds each process to all the CPUs in an "L1" cache. * "L2CACHE" binds each process to all the CPUs in an "L2" cache * "L3CACHE" binds each process to all the CPUs in an "L3" cache * "NUMA" binds each process to all the CPUs in a "NUMA" region * "PACKAGE" binds each process to all the CPUs in a "PACKAGE" Any directive can include qualifiers by adding a colon (:) and any combination of one or more of the following to the "--bind-to" option: * "OVERLOAD" indicates that objects can have more processes bound to them than CPUs within them * "IF-SUPPORTED" indicates that the job should continue to be launched and executed even if binding cannot be performed as requested. * "LIMIT=n" limits the number of processes bound to each eligible representative of the specified type to the given number. For example, speifying "--bind-to l3:limit=2" would direct PRRTE to bind ranks to the L3caches, limiting the number of processes bound to each l3cache to two - i.e., bind 2 processes to a given l3cache, and then move on to the next. Note: Directives and qualifiers are case-insensitive. "OVERLOAD" is the same as "overload". A more detailed description of the mapping, ranking, and binding procedure can be obtained via the "--help placement" option. prrte-4.1.1~rc2/src/mca/rmaps/base/Makefile.am0000664000175000017500000000300415203542132021253 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2015-2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ base/help-prte-rmaps-base.txt \ base/help-placement.txt \ base/help-bindto.txt \ base/help-mapby.txt \ base/help-rankby.txt headers += \ base/base.h \ base/rmaps_private.h libprtemca_rmaps_la_SOURCES += \ base/rmaps_base_frame.c \ base/rmaps_base_select.c \ base/rmaps_base_map_job.c \ base/rmaps_base_support_fns.c \ base/rmaps_base_ranking.c \ base/rmaps_base_print_fns.c \ base/rmaps_base_binding.c prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_ranking.c0000664000175000017500000004213215203542132023215 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "types.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" static void compute_app_rank(prte_job_t *jdata) { int i, j, k; prte_app_context_t *app; prte_proc_t *proc; for (i=0; i < jdata->apps->size; i++) { app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } k=0; /* loop thru all procs in job to find those from this app_context */ for (j=0; j < jdata->procs->size; j++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, j); if (NULL == proc) { continue; } if (proc->app_idx != app->idx) { continue; } proc->app_rank = k++; } } } static void compute_local_rank(prte_job_t *jdata) { int n, j, m; prte_node_t *node; prte_app_context_t *app; prte_proc_t *proc; pmix_rank_t lrank; for (n=0; n < jdata->map->nodes->size; n++) { node = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == node) { continue; } lrank = 0; for (j=0; j < jdata->apps->size; j++) { if (NULL == (app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, j))) { continue; } for (m=0; m < node->procs->size; m++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, m); if (NULL == proc) { continue; } if (!PMIX_CHECK_NSPACE(jdata->nspace, proc->name.nspace)) { continue; } if (proc->app_idx != app->idx) { continue; } proc->local_rank = lrank; ++lrank; } } } } int prte_rmaps_base_compute_vpids(prte_job_t *jdata, prte_rmaps_options_t *options) { int m, n, j, cnt; unsigned k, nobjs; prte_node_t *node; prte_proc_t *proc; int rc; hwloc_obj_t obj; pmix_rank_t rank; prte_app_context_t *app; bool one_found; if (options->userranked) { /* ranking has already been done, but we still need to * compute the local and app ranks (node rank is computed * on-the-fly during mapping) */ compute_local_rank(jdata); compute_app_rank(jdata); return PRTE_SUCCESS; } /* if ranking is by slot, then we assign ranks * sequentially across a node until that node * is fully ranked, and then move on to the next * node * * Node 0 Node 1 * Obj 0 Obj 1 Obj 0 Obj 1 * 0 1 2 3 8 9 10 11 * 4 5 6 7 12 13 14 15 */ if (PRTE_RANK_BY_SLOT == options->rank) { rank = 0; for (j=0; j < jdata->apps->size; j++) { if (NULL == (app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, j))) { continue; } for (n=0; n < jdata->map->nodes->size; n++) { node = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == node) { continue; } for (m=0; m < node->procs->size; m++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, m); if (NULL == proc) { continue; } /* ignore procs from other jobs */ if (!PMIX_CHECK_NSPACE(jdata->nspace, proc->name.nspace)) { continue; } /* ignore procs from other apps */ if (proc->app_idx != app->idx) { continue; } /* ignore procs that were already assigned */ if (PMIX_RANK_INVALID != proc->name.rank) { continue; } proc->name.rank = rank; PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(proc); return rc; } ++rank; } } } compute_local_rank(jdata); compute_app_rank(jdata); return PRTE_SUCCESS; } /* if we are ranking by NODE, then we cycle across * the nodes assigning a rank/node until we are * complete * * Node 0 Node 1 * Obj 0 Obj 1 Obj 0 Obj 1 * 0 2 4 6 1 3 5 7 * 8 10 12 14 9 11 13 15 */ if (PRTE_RANK_BY_NODE == options->rank) { rank = 0; for (j=0; j < jdata->apps->size; j++) { if (NULL == (app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, j))) { continue; } cnt = 0; one_found = true; while (cnt < app->num_procs && one_found) { one_found = false; for (n=0; n < jdata->map->nodes->size; n++) { node = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == node) { continue; } for (m=0; m < node->procs->size; m++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, m); if (NULL == proc) { continue; } /* ignore procs from other jobs */ if (!PMIX_CHECK_NSPACE(jdata->nspace, proc->name.nspace)) { continue; } /* ignore procs from other apps */ if (proc->app_idx != app->idx) { continue; } /* ignore procs that were already assigned */ if (PMIX_RANK_INVALID != proc->name.rank) { continue; } proc->name.rank = rank; PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(proc); return rc; } rank++; cnt++; one_found = true; break; // move on to next node } } } } compute_local_rank(jdata); compute_app_rank(jdata); return PRTE_SUCCESS; } /* if the ranking is fill, then we rank all the procs * within a given object before moving on to the next * * Node 0 Node 1 * Obj 0 Obj 1 Obj 0 Obj 1 * 0 1 4 5 8 9 12 13 * 2 3 6 7 10 11 14 15 */ if (PRTE_RANK_BY_FILL == options->rank) { rank = 0; for (j=0; j < jdata->apps->size; j++) { if (NULL == (app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, j))) { continue; } cnt = 0; for (n=0; n < jdata->map->nodes->size; n++) { node = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == node) { continue; } nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->maptype); if (0 == nobjs) { return PRTE_ERR_NOT_SUPPORTED; } /* for each object */ for (k=0; k < nobjs && cnt < app->num_procs; k++) { obj = prte_hwloc_base_get_obj_by_type(node->topology->topo, options->maptype, k); /* cycle thru the procs on this node */ for (m=0; m < node->procs->size && cnt < app->num_procs; m++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, m); if (NULL == proc) { continue; } /* ignore procs from other jobs */ if (!PMIX_CHECK_NSPACE(jdata->nspace, proc->name.nspace)) { continue; } /* ignore procs from other apps */ if (proc->app_idx != app->idx) { continue; } /* ignore procs that were already assigned */ if (PMIX_RANK_INVALID != proc->name.rank) { continue; } /* ignore procs not on this object */ if (obj != proc->obj) { continue; } /* this proc is on this object, so rank it */ proc->name.rank = rank; PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(proc); return rc; } rank++; cnt++; } } } } compute_local_rank(jdata); compute_app_rank(jdata); return PRTE_SUCCESS; } /* if the ranking is spanned, then we perform the * ranking as if it was one big node - i.e., we * rank one proc on each object, step to the next object * moving across all the nodes, then wrap around to the * first object on the first node. * * Node 0 Node 1 * Obj 0 Obj 1 Obj 0 Obj 1 * 0 4 1 5 2 6 3 7 * 8 12 9 13 10 14 11 15 */ if (PRTE_RANK_BY_SPAN == options->rank) { rank = 0; for (j=0; j < jdata->apps->size; j++) { if (NULL == (app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, j))) { continue; } cnt = 0; while (cnt < app->num_procs) { // scan across the nodes for (n=0; n < jdata->map->nodes->size; n++) { node = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == node) { continue; } // get number of this object type on this node nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->maptype); if (0 == nobjs) { return PRTE_ERR_NOT_SUPPORTED; } /* for each object */ for (k=0; k < nobjs && cnt < app->num_procs; k++) { /* get this object */ obj = prte_hwloc_base_get_obj_by_type(node->topology->topo, options->maptype, k); /* cycle thru the procs on this node */ for (m=0; m < node->procs->size && cnt < app->num_procs; m++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, m); if (NULL == proc) { continue; } /* ignore procs from other jobs */ if (!PMIX_CHECK_NSPACE(jdata->nspace, proc->name.nspace)) { continue; } /* ignore procs from other apps */ if (proc->app_idx != app->idx) { continue; } /* ignore procs that were already assigned */ if (PMIX_RANK_INVALID != proc->name.rank) { continue; } /* ignore procs not on this object */ if (obj != proc->obj) { continue; } proc->name.rank = rank; PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(proc); return rc; } ++rank; ++cnt; break; } } } } } compute_local_rank(jdata); compute_app_rank(jdata); return PRTE_SUCCESS; } /* cannot be anything else */ return PRTE_ERR_NOT_IMPLEMENTED; } /* when we restart a process on a different node, we have to * ensure that the node and local ranks assigned to the proc * don't overlap with any pre-existing proc on that node. If * we don't, then it would be possible for procs to conflict * when opening static ports, should that be enabled. */ void prte_rmaps_base_update_local_ranks(prte_job_t *jdata, prte_node_t *oldnode, prte_node_t *newnode, prte_proc_t *newproc) { int k; prte_node_rank_t node_rank; prte_local_rank_t local_rank; prte_proc_t *proc; PMIX_OUTPUT_VERBOSE((5, prte_rmaps_base_framework.framework_output, "%s rmaps:base:update_local_ranks", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* if the node hasn't changed, then we can just use the * pre-defined values */ if (oldnode == newnode) { return; } /* if the node has changed, then search the new node for the * lowest unused local and node rank */ node_rank = 0; retry_nr: for (k = 0; k < newnode->procs->size; k++) { /* if this proc is NULL, skip it */ if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(newnode->procs, k))) { continue; } if (node_rank == proc->node_rank) { node_rank++; goto retry_nr; } } newproc->node_rank = node_rank; local_rank = 0; retry_lr: for (k = 0; k < newnode->procs->size; k++) { /* if this proc is NULL, skip it */ if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(newnode->procs, k))) { continue; } /* ignore procs from other jobs */ if (!PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) { continue; } if (local_rank == proc->local_rank) { local_rank++; goto retry_lr; } } newproc->local_rank = local_rank; } prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_base_frame.c0000664000175000017500000010257315203542132022664 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/prte_cmd_line.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_show_help.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" /* * The following file was created by configure. It contains extern * statements and the definition of an array of pointers to each * component's public pmix_mca_base_component_t struct. */ #include "src/mca/rmaps/base/static-components.h" /* * Global variables */ prte_rmaps_base_t prte_rmaps_base = { .selected_modules = PMIX_LIST_STATIC_INIT, .mapping = 0, .ranking = 0, .ppr = NULL, .device = NULL, .inherit = false, .hwthread_cpus = false, .file = NULL, .default_pes = 0, .available = NULL, .baseset = NULL, .default_mapping_policy = NULL, .default_ranking_policy = NULL, .require_hwtcpus = false }; static int prte_rmaps_base_register(pmix_mca_base_register_flag_t flags) { int ret; PRTE_HIDE_UNUSED_PARAMS(flags); /* define default mapping policy */ prte_rmaps_base.default_mapping_policy = NULL; ret = pmix_mca_base_var_register("prte", NULL, NULL, "mapby", "Default mapping Policy [slot | hwthread | core | l1cache | " "l2cache | l3cache | numa | package | node | seq | dist | ppr | " "rankfile | pe-list=a,b (comma-delimited ranges of cpus to use for this job)]," " with supported colon-delimited modifiers: PE=y (for multiple cpus/proc), " "SPAN, OVERSUBSCRIBE, NOOVERSUBSCRIBE, NOLOCAL, HWTCPUS, CORECPUS, " "DEVICE=dev (for dist policy), INHERIT, NOINHERIT, ORDERED, FILE=%s (path to file containing sequential " "or rankfile entries). For more details, see \"prterun --help map-by\". " "The full directive need not be provided — " "only enough characters are required to uniquely identify the " "directive. Directive values are case insensitive", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_rmaps_base.default_mapping_policy); (void) pmix_mca_base_var_register_synonym(ret, "prte", NULL, NULL, "map_by", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "rmaps", "default", "mapping_policy", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); /* define default ranking policy */ prte_rmaps_base.default_ranking_policy = NULL; ret = pmix_mca_base_var_register("prte", NULL, NULL, "rankby", "Default ranking Policy [slot | node | span | fill]. " "For more details, see \"prterun --help rank-by\". The full " "directive need not be provided — only enough characters are " "required to uniquely identify the directive. Directive values " "are case insensitive", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_rmaps_base.default_ranking_policy); (void) pmix_mca_base_var_register_synonym(ret, "prte", NULL, NULL, "rank_by", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); (void) pmix_mca_base_var_register_synonym(ret, "prte", "rmaps", "default", "ranking_policy", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); prte_rmaps_base.inherit = false; (void) pmix_mca_base_var_register("prte", "rmaps", "default", "inherit", "Whether child jobs shall inherit mapping/ranking/binding " "directives from their parent by default", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_rmaps_base.inherit); return PRTE_SUCCESS; } static int prte_rmaps_base_close(void) { pmix_list_item_t *item; /* cleanup globals */ while (NULL != (item = pmix_list_remove_first(&prte_rmaps_base.selected_modules))) { PMIX_RELEASE(item); } PMIX_DESTRUCT(&prte_rmaps_base.selected_modules); hwloc_bitmap_free(prte_rmaps_base.available); hwloc_bitmap_free(prte_rmaps_base.baseset); return pmix_mca_base_framework_components_close(&prte_rmaps_base_framework, NULL); } /** * Function for finding and opening either all MCA components, or the one * that was specifically requested via a MCA parameter. */ static int prte_rmaps_base_open(pmix_mca_base_open_flag_t flags) { int rc; /* init the globals */ PMIX_CONSTRUCT(&prte_rmaps_base.selected_modules, pmix_list_t); prte_rmaps_base.available = hwloc_bitmap_alloc(); prte_rmaps_base.baseset = hwloc_bitmap_alloc(); /* set the default mapping and ranking policies */ if (NULL != prte_rmaps_base.default_mapping_policy) { rc = prte_rmaps_base_set_mapping_policy(NULL, prte_rmaps_base.default_mapping_policy); if (PRTE_SUCCESS != rc) { return rc; } } if (NULL != prte_rmaps_base.default_ranking_policy) { rc = prte_rmaps_base_set_ranking_policy(NULL, prte_rmaps_base.default_ranking_policy); if (PRTE_SUCCESS != rc) { return rc; } } /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_rmaps_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, rmaps, "PRTE Mapping Subsystem", prte_rmaps_base_register, prte_rmaps_base_open, prte_rmaps_base_close, prte_rmaps_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); PMIX_CLASS_INSTANCE(prte_rmaps_base_selected_module_t, pmix_list_item_t, NULL, NULL); static int check_modifiers(char *ck, prte_job_t *jdata, prte_mapping_policy_t *tmp) { char **ck2, *ptr; int i; uint16_t u16; bool inherit_given = false; bool noinherit_given = false; bool hwthread_cpus_given = false; bool core_cpus_given = false; bool oversubscribe_given = false; bool nooversubscribe_given = false; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "%s rmaps:base check modifiers with %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == ck) ? "NULL" : ck); if (NULL == ck) { return PRTE_SUCCESS; } ck2 = PMIx_Argv_split(ck, ':'); for (i = 0; NULL != ck2[i]; i++) { if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_SPAN)) { PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_SPAN); PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_GIVEN); } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_OVERSUB)) { if (nooversubscribe_given) { /* conflicting directives */ pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "OVERSUBSCRIBE", "NOOVERSUBSCRIBE"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } PRTE_UNSET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_NO_OVERSUBSCRIBE); PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_SUBSCRIBE_GIVEN); oversubscribe_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_NOOVER)) { if (oversubscribe_given) { /* conflicting directives */ pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "OVERSUBSCRIBE", "NOOVERSUBSCRIBE"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_NO_OVERSUBSCRIBE); PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_SUBSCRIBE_GIVEN); nooversubscribe_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_NOLOCAL)) { PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_NO_USE_LOCAL); } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_ORDERED)) { if (NULL == jdata) { pmix_show_help("help-prte-rmaps-base.txt", "unsupported-default-modifier", true, "mapping policy", ck2[i]); return PRTE_ERR_SILENT; } PRTE_SET_MAPPING_DIRECTIVE(*tmp, PRTE_MAPPING_ORDERED); } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_PE)) { /* Numeric value must immediately follow '=' (PE=2) */ u16 = strtol(&ck2[i][3], &ptr, 10); if ('\0' != *ptr) { /* missing the value or value is invalid */ pmix_show_help("help-prte-rmaps-base.txt", "invalid-value", true, "mapping policy", "PE", ck2[i]); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (NULL == jdata) { prte_rmaps_base.default_pes = u16; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, PRTE_ATTR_GLOBAL, &u16, PMIX_UINT16); } } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_INHERIT)) { if (noinherit_given) { /* conflicting directives */ pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "INHERIT", "NOINHERIT"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (NULL == jdata) { prte_rmaps_base.inherit = true; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_INHERIT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } inherit_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_NOINHERIT)) { if (inherit_given) { /* conflicting directives */ pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "INHERIT", "NOINHERIT"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (NULL == jdata) { prte_rmaps_base.inherit = false; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_NOINHERIT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } noinherit_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_HWTCPUS)) { if (core_cpus_given) { pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "HWTCPUS", "CORECPUS"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (NULL == jdata) { prte_rmaps_base.hwthread_cpus = true; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } hwthread_cpus_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_CORECPUS)) { if (hwthread_cpus_given) { pmix_show_help("help-prte-rmaps-base.txt", "conflicting-directives", true, "HWTCPUS", "CORECPUS"); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (prte_rmaps_base.require_hwtcpus) { if (NULL == jdata) { prte_rmaps_base.hwthread_cpus = true; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } else { if (NULL == jdata) { prte_rmaps_base.hwthread_cpus = false; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_CORE_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } core_cpus_given = true; } else if (PMIX_CHECK_CLI_OPTION(ck2[i], PRTE_CLI_QFILE)) { if ('\0' == ck2[i][5]) { /* missing the value */ pmix_show_help("help-prte-rmaps-base.txt", "missing-value", true, "mapping policy", "FILE", ck2[i]); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } if (NULL == jdata) { if (NULL != prte_rmaps_base.file) { // cannot specify it twice pmix_show_help("help-prte-rmaps-base.txt", "multiply-defined", true, "mapping policy", "FILE", prte_rmaps_base.file, ck2[i]); PMIx_Argv_free(ck2); return PRTE_ERR_SILENT; } prte_rmaps_base.file = strdup(&ck2[i][5]); } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_FILE, PRTE_ATTR_GLOBAL, &ck2[i][5], PMIX_STRING); } } else { /* unrecognized modifier */ PMIx_Argv_free(ck2); return PRTE_ERR_BAD_PARAM; } } PMIx_Argv_free(ck2); return PRTE_SUCCESS; } int prte_rmaps_base_set_default_mapping(prte_job_t *jdata, prte_rmaps_options_t *options) { prte_binding_policy_t bind; if (1 < options->cpus_per_rank) { /* assigning multiple cpus to a rank requires that we map to * objects that have multiple cpus in them, so default * to byslot if nothing else was specified by the user. */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps mapping not given with multiple cpus/rank - using byslot"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYSLOT); } else if (PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { /* they specified a binding policy, but not a mapping policy, so * map by the binding object */ bind = PRTE_GET_BINDING_POLICY(jdata->map->binding); switch(bind) { case PRTE_BIND_TO_NONE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYSLOT); break; case PRTE_BIND_TO_PACKAGE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYPACKAGE); break; case PRTE_BIND_TO_NUMA: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYNUMA); break; case PRTE_BIND_TO_L3CACHE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYL3CACHE); break; case PRTE_BIND_TO_L2CACHE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYL2CACHE); break; case PRTE_BIND_TO_L1CACHE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYL1CACHE); break; case PRTE_BIND_TO_CORE: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYCORE); break; case PRTE_BIND_TO_HWTHREAD: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYHWTHREAD); break; default: PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYSLOT); break; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps mapping not given but binding set - using %s", prte_rmaps_base_print_mapping(jdata->map->mapping)); } else { if (options->nprocs <= 2) { // map by whatever CPU we are using if (options->use_hwthreads) { /* if we are using hwthread cpus, then map by those */ pmix_output_verbose(options->verbosity, options->stream, "mca:rmaps mapping not given - using byhwthread"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYHWTHREAD); } else { /* otherwise map by core */ pmix_output_verbose(options->verbosity, options->stream, "mca:rmaps mapping not given - using bycore"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYCORE); } } else { /* if package is available, map by that */ if (NULL != prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_PACKAGE, 0)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps mapping not set by user - using bypackage"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYPACKAGE); } else if (NULL != prte_hwloc_base_get_obj_by_type(prte_hwloc_topology, HWLOC_OBJ_NUMANODE, 0)) { /* if NUMA is available, map by that */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps mapping not set by user - using bynuma"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYNUMA); } else { /* if we have neither, then just do by slot */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps mapping not given and no packages/NUMAs - using byslot"); PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYSLOT); } } } return PRTE_SUCCESS; } int prte_rmaps_base_set_mapping_policy(prte_job_t *jdata, char *inspec) { char **ck, **range; char *ptr, *cptr, *val; prte_mapping_policy_t tmp; int rc, n; bool ppr = false; char *temp_token, *parm_delimiter; /* set defaults */ tmp = 0; pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "%s rmaps:base set policy with %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == inspec) ? "NULL" : inspec); if (NULL == inspec) { return PRTE_SUCCESS; } /* see if a colon was included - if so, then we have a modifier */ ck = PMIx_Argv_split(inspec, ':'); if (1 < PMIx_Argv_count(ck)) { if (0 == strcasecmp(ck[0], "ppr")) { if (4 < pmix_output_get_verbosity(prte_rmaps_base_framework.framework_output)) { pmix_asprintf(&cptr, "%s:%s", ck[1], ck[2]); pmix_output(prte_rmaps_base_framework.framework_output, "%s rmaps:base policy %s modifiers %s provided", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), ck[0], cptr); free(cptr); } /* at this point, ck contains at least * two fields (specifying the #procs/obj and the object we are * to map by). we have to allow additional modifiers here - e.g., * specifying #pe's/proc or oversubscribe - so check for modifiers. if * they are present, the rest of ck will look like "N:obj:mod1,mod2,mod3" */ if (3 > PMIx_Argv_count(ck)) { /* this is an error - there had to be at least one * colon to delimit the number from the object type */ pmix_show_help("help-prte-rmaps-base.txt", "invalid-pattern", true, inspec); PMIx_Argv_free(ck); return PRTE_ERR_SILENT; } /* at this point, ck[1] contains the number of procs/resource, * ck[2] contains the string that describes * the object and ck[3] is either NULL or contains any * modifiers (i.e., "#:obj:mod1,mod2") */ /* save the pattern */ pmix_asprintf(&cptr, "%s:%s", ck[1], ck[2]); if (NULL == jdata) { prte_rmaps_base.ppr = cptr; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_PPR, PRTE_ATTR_GLOBAL, cptr, PMIX_STRING); free(cptr); } PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_PPR); PRTE_SET_MAPPING_DIRECTIVE(tmp, PRTE_MAPPING_GIVEN); ppr = true; if (NULL == ck[3]) { /* there are no modifiers, so we are done */ PMIx_Argv_free(ck); goto setpolicy; } PMIX_ARGV_JOIN(cptr, &ck[3], ':'); } else { PMIX_ARGV_JOIN(cptr, &ck[1], ':'); pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "%s rmaps:base policy %s modifiers %s provided", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), ck[0], cptr); } if (PRTE_SUCCESS != (rc = check_modifiers(cptr, jdata, &tmp)) && PRTE_ERR_TAKE_NEXT_OPTION != rc) { if (PRTE_ERR_BAD_PARAM == rc) { pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-modifier", true, inspec); rc = PRTE_ERR_SILENT; } PMIx_Argv_free(ck); free(cptr); return rc; } if (ppr) { /* we are done */ PMIx_Argv_free(ck); goto setpolicy; } } /* we get here if there was only one element in * the ck array. However, if the input string * started with a ':', then there is no specified * mapping policy for us to set, but we still * have to process the modifier */ if (':' == inspec[0]) { if (PRTE_SUCCESS != (rc = check_modifiers(&inspec[1], jdata, &tmp)) && PRTE_ERR_TAKE_NEXT_OPTION != rc) { if (PRTE_ERR_BAD_PARAM == rc) { pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-modifier", true, inspec); rc = PRTE_ERR_SILENT; } PMIx_Argv_free(ck); return rc; } PMIx_Argv_free(ck); goto setpolicy; } cptr = ck[0]; // check for an '=' and split out the value from the option if (NULL != (ptr = strchr(ck[0], '='))) { *ptr = '\0'; cptr = strdup(ck[0]); *ptr = '='; // restore the option ++ptr; if ('\0' == *ptr) { /* malformed option */ pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-policy", true, "mapping", ck[0]); PMIx_Argv_free(ck); free(cptr); return PRTE_ERR_SILENT; } val = strdup(ptr); } else { cptr = strdup(ck[0]); val = NULL; } if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_SLOT)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYSLOT); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_NODE)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYNODE); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_SEQ)) { /* there are several mechanisms by which the file specifying * the sequence can be passed, so not really feasible to check * it here */ PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_SEQ); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_CORE)) { if (prte_rmaps_base.require_hwtcpus) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYHWTHREAD); } else { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYCORE); } } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_L1CACHE)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYL1CACHE); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_L2CACHE)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYL2CACHE); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_L3CACHE)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYL3CACHE); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_NUMA)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYNUMA); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_PACKAGE)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYPACKAGE); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_RANKFILE)) { /* check that the file was given */ if ((NULL == jdata && NULL == prte_rmaps_base.file) || (NULL != jdata && !prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, NULL, PMIX_STRING))) { pmix_show_help("help-prte-rmaps-base.txt", "rankfile-no-filename", true); PMIx_Argv_free(ck); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_BAD_PARAM; } /* if they asked for rankfile and didn't specify one, but did * provide one via MCA param, then use it */ if (NULL != jdata) { if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, NULL, PMIX_STRING)) { if (NULL == prte_rmaps_base.file) { /* also not allowed */ pmix_show_help("help-prte-rmaps-base.txt", "rankfile-no-filename", true); PMIx_Argv_free(ck); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_BAD_PARAM; } prte_set_attribute(&jdata->attributes, PRTE_JOB_FILE, PRTE_ATTR_GLOBAL, prte_rmaps_base.file, PMIX_STRING); } } PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYUSER); } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_HWT)) { PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_BYHWTHREAD); /* if we are mapping processes to individual hwthreads, then * we need to treat those hwthreads as separate cpus */ if (NULL == jdata) { prte_rmaps_base.hwthread_cpus = true; } else { prte_set_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } else if (PMIX_CHECK_CLI_OPTION(cptr, PRTE_CLI_PELIST)) { if (NULL == jdata) { pmix_show_help("help-prte-rmaps-base.txt", "unsupported-default-policy", true, "mapping", cptr); PMIx_Argv_free(ck); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_SILENT; } if (NULL == val) { /* malformed option */ pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-policy", true, "mapping", ck[0]); PMIx_Argv_free(ck); free(cptr); return PRTE_ERR_SILENT; } /* Verify the list is composed of comma-delimited ranges */ temp_token = strtok(val, ","); while (NULL != temp_token) { // check for range range = PMIx_Argv_split(temp_token, '-'); if (2 < PMIx_Argv_count(range)) { // can only have one '-' delimiter pmix_show_help("help-prte-rmaps-base.txt", "invalid-value", true, "mapping", "PE-LIST", ck[0]); PMIx_Argv_free(ck); PMIx_Argv_free(range); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_SILENT; } for (n=0; NULL != range[n]; n++) { (void)strtol(range[n], &parm_delimiter, 10); if ('\0' != *parm_delimiter) { pmix_show_help("help-prte-rmaps-base.txt", "invalid-value", true, "mapping", "PE-LIST", val); PMIx_Argv_free(ck); PMIx_Argv_free(range); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_SILENT; } } PMIx_Argv_free(range); temp_token = strtok(NULL, ","); } prte_set_attribute(&jdata->attributes, PRTE_JOB_CPUSET, PRTE_ATTR_GLOBAL, ptr, PMIX_STRING); PRTE_SET_MAPPING_POLICY(tmp, PRTE_MAPPING_PELIST); PRTE_SET_MAPPING_DIRECTIVE(tmp, PRTE_MAPPING_GIVEN); } else { pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-policy", true, "mapping", cptr); PMIx_Argv_free(ck); free(cptr); if (NULL != val) { free(val); } return PRTE_ERR_SILENT; } PMIx_Argv_free(ck); free(cptr); if (NULL != val) { free(val); } PRTE_SET_MAPPING_DIRECTIVE(tmp, PRTE_MAPPING_GIVEN); setpolicy: if (NULL == jdata) { prte_rmaps_base.mapping = tmp; } else { if (NULL == jdata->map) { PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } jdata->map->mapping = tmp; } return PRTE_SUCCESS; } int prte_rmaps_base_set_default_ranking(prte_job_t *jdata, prte_rmaps_options_t *options) { int rc; PRTE_HIDE_UNUSED_PARAMS(options); rc = prte_rmaps_base_set_ranking_policy(jdata, NULL); return rc; } int prte_rmaps_base_set_ranking_policy(prte_job_t *jdata, char *spec) { prte_ranking_policy_t tmp; prte_mapping_policy_t mapping; /* set default */ tmp = 0; if (NULL == spec) { if (NULL == jdata) { return PRTE_SUCCESS; } if (NULL == jdata->map) { jdata->map = PMIX_NEW(prte_job_map_t); } if (PRTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) { return PRTE_SUCCESS; } mapping = PRTE_GET_MAPPING_POLICY(jdata->map->mapping); /* if mapping by-node, then default to rank-by node */ if (PRTE_MAPPING_BYNODE == mapping) { PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_NODE); } else if (PRTE_MAPPING_BYSLOT == mapping) { /* default to by-slot */ PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_SLOT); } else if (PRTE_MAPPING_SPAN & PRTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) { /* default to by-span */ PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_SPAN); } else if (PRTE_MAPPING_BYHWTHREAD >= mapping && PRTE_MAPPING_BYNUMA <= mapping) { /* default to by-slot */ PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_FILL); } else { /* default to slot */ PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_SLOT); } jdata->map->ranking = tmp; return PRTE_SUCCESS; } if (PMIX_CHECK_CLI_OPTION(spec, PRTE_CLI_SLOT)) { PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_SLOT); } else if (PMIX_CHECK_CLI_OPTION(spec, PRTE_CLI_NODE)) { PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_NODE); } else if (PMIX_CHECK_CLI_OPTION(spec, PRTE_CLI_FILL)) { PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_FILL); } else if (PMIX_CHECK_CLI_OPTION(spec, PRTE_CLI_SPAN)) { PRTE_SET_RANKING_POLICY(tmp, PRTE_RANK_BY_SPAN); } else { pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-policy", true, "ranking", spec); return PRTE_ERR_SILENT; } PRTE_SET_RANKING_DIRECTIVE(tmp, PRTE_RANKING_GIVEN); if (NULL == jdata) { prte_rmaps_base.ranking = tmp; } else { if (NULL == jdata->map) { jdata->map = PMIX_NEW(prte_job_map_t); } jdata->map->ranking = tmp; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/base/rmaps_private.h0000664000175000017500000000564015203542132022254 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_RMAPS_PRIVATE_H #define PRTE_MCA_RMAPS_PRIVATE_H /* * includes */ #include "prte_config.h" #include "types.h" #include "src/hwloc/hwloc-internal.h" #include "src/runtime/prte_globals.h" #include "src/mca/schizo/schizo.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS /* * Base API functions */ /* LOCAL FUNCTIONS for use by RMAPS components */ PRTE_EXPORT int prte_rmaps_base_get_target_nodes(pmix_list_t *node_list, int32_t *total_num_slots, prte_job_t *jdata, prte_app_context_t *app, prte_mapping_policy_t policy, bool initial_map, bool silent, bool keepall); PRTE_EXPORT prte_proc_t *prte_rmaps_base_setup_proc(prte_job_t *jdata, prte_app_idx_t idx, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options); PRTE_EXPORT void prte_rmaps_base_get_starting_point(pmix_list_t *node_list, prte_job_t *jdata); PRTE_EXPORT int prte_rmaps_base_compute_vpids(prte_job_t *jdata, prte_rmaps_options_t *options); PRTE_EXPORT int prte_rmaps_base_bind_proc(prte_job_t *jdata, prte_proc_t *proc, prte_node_t *node, hwloc_obj_t obj, prte_rmaps_options_t *options); PRTE_EXPORT void prte_rmaps_base_update_local_ranks(prte_job_t *jdata, prte_node_t *oldnode, prte_node_t *newnode, prte_proc_t *newproc); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/Makefile.am0000664000175000017500000000242415203542132020346 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_rmaps.la libprtemca_rmaps_la_SOURCES = # pkgdata setup EXTRA_DIST = # local files headers = rmaps.h rmaps_types.h libprtemca_rmaps_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/rmaps/rmaps_types.h0000664000175000017500000001412315203542132021030 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_RMAPS_TYPES_H #define PRTE_MCA_RMAPS_TYPES_H #include "prte_config.h" #include "constants.h" #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/runtime/prte_globals.h" /* * General MAP types - instanced in runtime/prte_globals_class_instances.h */ BEGIN_C_DECLS typedef uint16_t prte_mapping_policy_t; #define PRTE_MAPPING_POLICY PRTE_UINT16 typedef uint16_t prte_ranking_policy_t; #define PRTE_RANKING_POLICY PRTE_UINT16 /* * Structure that represents the mapping of a job to an * allocated set of resources. */ struct prte_job_map_t { pmix_object_t super; /* user-specified mapping params */ char *req_mapper; /* requested mapper */ char *last_mapper; /* last mapper used */ prte_mapping_policy_t mapping; prte_ranking_policy_t ranking; prte_binding_policy_t binding; bool rtos_set; /* *** */ /* number of new daemons required to be launched * to support this job map */ int32_t num_new_daemons; /* starting vpid of the new daemons - they will * be sequential from that point */ pmix_rank_t daemon_vpid_start; /* number of nodes participating in this job */ int32_t num_nodes; /* array of pointers to nodes in this map for this job */ pmix_pointer_array_t *nodes; }; typedef struct prte_job_map_t prte_job_map_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_job_map_t); typedef struct { /* input info */ uint16_t cpus_per_rank; bool use_hwthreads; int stream; int verbosity; char *cpuset; hwloc_cpuset_t job_cpuset; bool bindsupport; bool donotlaunch; bool membind_warned; bool oversubscribe; bool overload; /* mapping values */ prte_mapping_policy_t map; bool mapspan; bool ordered; prte_binding_policy_t mapdepth; unsigned ncpus; int nprocs; hwloc_obj_type_t maptype; /* #procs/resource as per PPR */ int pprn; /* ranking values */ prte_ranking_policy_t rank; bool userranked; unsigned nnodes; unsigned total_nobjs; unsigned nobjs; /* binding values */ prte_binding_policy_t bind; bool dobind; hwloc_obj_type_t hwb; uint16_t limit; /* usage tracking */ hwloc_cpuset_t target; hwloc_obj_t obj; } prte_rmaps_options_t; /* ** * Macro for use in components that are of type rmaps */ #define PRTE_RMAPS_BASE_VERSION_4_0_0 PRTE_MCA_BASE_VERSION_3_0_0("rmaps", 4, 0, 0) /* define map-related directives */ #define PRTE_MAPPING_NO_USE_LOCAL 0x0100 #define PRTE_MAPPING_NO_OVERSUBSCRIBE 0x0200 #define PRTE_MAPPING_SUBSCRIBE_GIVEN 0x0400 #define PRTE_MAPPING_SPAN 0x0800 #define PRTE_MAPPING_ORDERED 0x1000 /* directives given */ #define PRTE_MAPPING_LOCAL_GIVEN 0x2000 #define PRTE_MAPPING_GIVEN 0x4000 /* macros to set/get/unset mapping directive */ #define PRTE_SET_MAPPING_DIRECTIVE(target, pol) (target) |= (pol) #define PRTE_UNSET_MAPPING_DIRECTIVE(target, pol) (target) &= ~(pol) #define PRTE_GET_MAPPING_DIRECTIVE(pol) ((pol) &0xff00) /* round-robin policies */ /* start with hardware-based options * so the values match the corresponding * levels in src/hwloc/hwloc-internal.h */ #define PRTE_MAPPING_BYNODE 1 #define PRTE_MAPPING_BYNUMA 2 #define PRTE_MAPPING_BYPACKAGE 3 #define PRTE_MAPPING_BYL3CACHE 4 #define PRTE_MAPPING_BYL2CACHE 5 #define PRTE_MAPPING_BYL1CACHE 6 #define PRTE_MAPPING_BYCORE 7 #define PRTE_MAPPING_BYHWTHREAD 8 /* now take the other round-robin options */ #define PRTE_MAPPING_BYSLOT 9 #define PRTE_MAPPING_BYDIST 10 #define PRTE_MAPPING_PELIST 11 /* convenience - declare anything <= 15 to be round-robin*/ #define PRTE_MAPPING_RR 16 /* sequential policy */ #define PRTE_MAPPING_SEQ 20 #define PRTE_MAPPING_COLOCATE 21 /* rank file and other user-defined mapping */ #define PRTE_MAPPING_BYUSER 22 /* pattern-based mapping */ #define PRTE_MAPPING_PPR 23 /* macro to separate out the mapping policy * from the directives */ #define PRTE_GET_MAPPING_POLICY(pol) ((pol) & 0x00ff) /* macro to determine if mapping policy is set */ #define PRTE_MAPPING_POLICY_IS_SET(pol) ((pol) & 0x00ff) #define PRTE_SET_MAPPING_POLICY(target, pol) (target) = (pol) | ((target) & 0xff00) /* define ranking directives */ #define PRTE_RANKING_GIVEN 0x1000 #define PRTE_SET_RANKING_DIRECTIVE(target, pol) (target) |= (pol) #define PRTE_UNSET_RANKING_DIRECTIVE(target, pol) (target) &= ~(pol) #define PRTE_GET_RANKING_DIRECTIVE(pol) ((pol) & 0xf000) /* define ranking policies */ #define PRTE_RANK_BY_NODE 1 #define PRTE_RANK_BY_SLOT 2 #define PRTE_RANK_BY_FILL 3 #define PRTE_RANK_BY_SPAN 4 #define PRTE_RANKING_BYUSER 5 #define PRTE_GET_RANKING_POLICY(pol) ((pol) &0x0fff) /* macro to determine if ranking policy is set */ #define PRTE_RANKING_POLICY_IS_SET(pol) ((pol) & 0x0fff) #define PRTE_SET_RANKING_POLICY(target, pol) (target) = (pol) | ((target) & 0xf000) END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/rank_file/0000775000175000017500000000000015203542132020242 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/rank_file/common_sym_whitelist.txt0000664000175000017500000000026115203542132025256 0ustar alastairalastair# Ignore symbols in this component that are auto-generated and we # can't do anything about them (e.g., flex/bison symbols). prte_rmaps_rank_file_leng prte_rmaps_rank_file_text prrte-4.1.1~rc2/src/mca/rmaps/rank_file/owner.txt0000664000175000017500000000024215203542132022133 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: IBM status: maintenance prrte-4.1.1~rc2/src/mca/rmaps/rank_file/help-rmaps_rank_file.txt0000664000175000017500000000446115203542132025072 0ustar alastairalastair# Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2018-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2026 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for rankle utilities. # [no-rankfile] %s was unable to open the rankfile: Filename: %s Check to make sure the path and filename are correct. usage: %s --map-by rankfile:file= ./app #include#prrte#help-placement.txt#placement-rankfiles # [bad-host] The rankfile that was used claimed that a host was either not allocated or oversubscribed its slots. Please review your rank-slot assignments and your host allocation to ensure a proper match. Also, some systems may require using full hostnames, such as "host1.example.com" (instead of just plain "host1"). Host: %s # [bad-index] Rankfile claimed host %s by index that is bigger than number of allocated hosts. # [bad-assign] Error, rank %d is already assigned to %s, check %s # [bad-syntax] Error, invalid syntax in the rankfile (%s) syntax must be the following rank i=host_i slot=string Examples of proper syntax include: rank 1=host1 slot=1:0,1 rank 0=host2 slot=0:* rank 2=host4 slot=1-2 rank 3=host3 slot=0:1;1:0-2 # [missing-rank] A rank is missing its location specification: Rank: %d Rank file: %s All processes must have their location specified in the rank file. Either add an entry to the file, or provide a default slot_list to use for any unspecified ranks. # [missing-cpu] While parsing the rankfile, %s encountered a line that specified a non-existent CPU: Slots: %s Available CPUs: %s Please correct the line and try again. # [rmaps:proc-slots-overloaded] A process attempted to bind to resources already allocated on a node to a different process. Process : %s Node : %s Process Requested CPU Set : %s Node Available CPU Set : %s Overlapping CPU Set : %s If this is intentional then you must pass the "overload-allowed" qualifier to the --bind-to option. --bind-to :overload-allowed prrte-4.1.1~rc2/src/mca/rmaps/rank_file/rmaps_rank_file_lex.h0000664000175000017500000000455615203542132024431 0ustar alastairalastair/* -*- C -*- * * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_MCA_RANK_FILE_RANKFILE_LEX_H_ #define PRTE_MCA_RANK_FILE_RANKFILE_LEX_H_ #include "prte_config.h" #ifdef malloc # undef malloc #endif #ifdef realloc # undef realloc #endif #ifdef free # undef free #endif #include BEGIN_C_DECLS typedef union { int ival; char *sval; } prte_rmaps_rank_file_value_t; extern int prte_rmaps_rank_file_lex(void); extern FILE *prte_rmaps_rank_file_in; extern int prte_rmaps_rank_file_line; extern bool prte_rmaps_rank_file_done; extern prte_rmaps_rank_file_value_t prte_rmaps_rank_file_value; int prte_rmaps_rank_file_wrap(void); /* * Make lex-generated files not issue compiler warnings */ #define YY_STACK_USED 0 #define YY_ALWAYS_INTERACTIVE 0 #define YY_NEVER_INTERACTIVE 0 #define YY_MAIN 0 #define YY_NO_UNPUT 1 #define YY_SKIP_YYWRAP 1 #define PRTE_RANKFILE_DONE 0 #define PRTE_RANKFILE_ERROR 1 #define PRTE_RANKFILE_QUOTED_STRING 2 #define PRTE_RANKFILE_EQUAL 3 #define PRTE_RANKFILE_INT 4 #define PRTE_RANKFILE_STRING 5 #define PRTE_RANKFILE_RANK 6 #define PRTE_RANKFILE_USERNAME 10 #define PRTE_RANKFILE_IPV4 11 #define PRTE_RANKFILE_HOSTNAME 12 #define PRTE_RANKFILE_NEWLINE 13 #define PRTE_RANKFILE_IPV6 14 #define PRTE_RANKFILE_SLOT 15 #define PRTE_RANKFILE_RELATIVE 16 END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/rank_file/rmaps_rank_file_component.c0000664000175000017500000000572515203542132025635 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_base.h" #include "src/util/pmix_show_help.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" #include "src/mca/rmaps/rank_file/rmaps_rank_file.h" #include "src/mca/rmaps/rank_file/rmaps_rank_file_lex.h" /* * Local functions */ static int prte_rmaps_rank_file_query(pmix_mca_base_module_t **module, int *priority); prte_rmaps_rf_component_t prte_mca_rmaps_rank_file_component = { .super = { PRTE_RMAPS_BASE_VERSION_4_0_0, .pmix_mca_component_name = "rank_file", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_query_component = prte_rmaps_rank_file_query, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, rmaps, rank_file) static int prte_rmaps_rank_file_query(pmix_mca_base_module_t **module, int *priority) { /* Set the rankfile priority to be LESS than the lsf component * so that the LSF component can decide if it needs to run */ *priority = 95; *module = (pmix_mca_base_module_t *) &prte_rmaps_rank_file_module; return PRTE_SUCCESS; } static void rf_map_construct(prte_rmaps_rank_file_map_t *ptr) { ptr->node_name = NULL; memset(ptr->slot_list, (char) 0x00, RMAPS_RANK_FILE_MAX_SLOTS); } static void rf_map_destruct(prte_rmaps_rank_file_map_t *ptr) { if (NULL != ptr->node_name) free(ptr->node_name); } PMIX_CLASS_INSTANCE(prte_rmaps_rank_file_map_t, pmix_object_t, rf_map_construct, rf_map_destruct); prrte-4.1.1~rc2/src/mca/rmaps/rank_file/Makefile.am0000664000175000017500000000447615203542132022311 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008 Voltaire. All rights reserved # # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-rmaps_rank_file.txt AM_LFLAGS = -Pprte_rmaps_rank_file_ LEX_OUTPUT_ROOT = lex.prte_rmaps_rank_file_ # we do NOT want picky compilers down here due to flex CFLAGS = $(PMIX_CFLAGS_BEFORE_PICKY) sources = \ rmaps_rank_file.c \ rmaps_rank_file.h \ rmaps_rank_file_component.c \ rmaps_rank_file_lex.l \ rmaps_rank_file_lex.h # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_rmaps_rank_file_DSO component_noinst = component_install = prte_mca_rmaps_rank_file.la else component_noinst = libprtemca_rmaps_rank_file.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_rmaps_rank_file_la_SOURCES = $(sources) prte_mca_rmaps_rank_file_la_LDFLAGS = -module -avoid-version prte_mca_rmaps_rank_file_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_rmaps_rank_file_la_SOURCES =$(sources) libprtemca_rmaps_rank_file_la_LDFLAGS = -module -avoid-version maintainer-clean-local: rm -f rmaps_rank_file_lex.c prrte-4.1.1~rc2/src/mca/rmaps/rank_file/rmaps_rank_file_lex.l0000664000175000017500000001241515203542132024426 0ustar alastairalastair%option nounput %option noinput %{ /* -*- C -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H #include #endif #include "src/mca/rmaps/rank_file/rmaps_rank_file_lex.h" #include "src/util/pmix_output.h" BEGIN_C_DECLS int prte_rmaps_rank_file_yywrap(void); END_C_DECLS /* * global variables */ int prte_rmaps_rank_file_line=1; prte_rmaps_rank_file_value_t prte_rmaps_rank_file_value = {0}; bool prte_rmaps_rank_file_done = false; #ifndef YY_BREAK #define YY_BREAK #endif %} WHITE [\f\t\v ] %x comment %% {WHITE}*\n { prte_rmaps_rank_file_line++; return PRTE_RANKFILE_NEWLINE; } #.*\n { prte_rmaps_rank_file_line++; return PRTE_RANKFILE_NEWLINE; } "//".*\n { prte_rmaps_rank_file_line++; return PRTE_RANKFILE_NEWLINE; } "/*" { BEGIN(comment); return PRTE_RANKFILE_NEWLINE; } [^*\n]* { break; } /* Eat up non '*'s */ "*"+[^*/\n]* { break; } /* Eat '*'s not followed by a '/' */ \n { prte_rmaps_rank_file_line++; return PRTE_RANKFILE_NEWLINE; } "*"+"/" { BEGIN(INITIAL); /* Done with Block Comment */ return PRTE_RANKFILE_NEWLINE; } \"[^\"]*\" { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_QUOTED_STRING; } {WHITE}+ { break; } "=" { return PRTE_RANKFILE_EQUAL; } rank { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_RANK; } slot { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_SLOT; } slots { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_SLOT; } username { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_USERNAME; } "user-name" { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_USERNAME; } "user_name" { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_USERNAME; } [0-9]+ { prte_rmaps_rank_file_value.ival = atol(yytext); return PRTE_RANKFILE_INT; } %{ /* First detect hosts as standard Strings (but without ".") * then username@IPv4 or IPV4, then username@IPv6 or IPv6, * followed by username@hostname or hostname */ %} [A-Za-z0-9_\-,\;:*@]* { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_STRING; } ([A-Za-z0-9][A-Za-z0-9_\-]*"@")?([0-9]{1,3}"."){3}[0-9]{1,3} { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_IPV4; } ([A-Za-z0-9][A-Za-z0-9_\-]*"@")?([A-Fa-f0-9]{0,4}":")+[":"]*([A-Fa-f0-9]{0,4}":")+[A-Fa-f0-9]{1,4} { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_IPV6; } ([A-Za-z0-9][A-Za-z0-9_\-]*"@")?[A-Za-z][A-Za-z0-9_\-\.]* { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_HOSTNAME; } \+n[0-9]+ { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_RELATIVE; } . { prte_rmaps_rank_file_value.sval = yytext; return PRTE_RANKFILE_ERROR; } %% /* Old flex (2.5.4a? and older) does not define a destroy function */ #if !defined(YY_FLEX_SUBMINOR_VERSION) #define YY_FLEX_SUBMINOR_VERSION 0 #endif #if (YY_FLEX_MAJOR_VERSION < 2) || (YY_FLEX_MAJOR_VERSION == 2 && (YY_FLEX_MINOR_VERSION < 5 || (YY_FLEX_MINOR_VERSION == 5 && YY_FLEX_SUBMINOR_VERSION < 5))) int prte_rmaps_rank_file_lex_destroy (void) { if (NULL != YY_CURRENT_BUFFER) { yy_delete_buffer(YY_CURRENT_BUFFER); #if defined(YY_CURRENT_BUFFER_LVALUE) YY_CURRENT_BUFFER_LVALUE = NULL; #else YY_CURRENT_BUFFER = NULL; #endif /* YY_CURRENT_BUFFER_LVALUE */ } return YY_NULL; } #endif int prte_rmaps_rank_file_wrap(void) { prte_rmaps_rank_file_done = true; return 1; } prrte-4.1.1~rc2/src/mca/rmaps/rank_file/rmaps_rank_file.h0000664000175000017500000000411315203542132023546 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * Copyright (c) 2022 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Mapping */ #ifndef PRTE_RMAPS_RF_H #define PRTE_RMAPS_RF_H #include "prte_config.h" #include "src/class/pmix_object.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS #define RMAPS_RANK_FILE_MAX_SLOTS 64 int prte_rmaps_rank_file_lex_destroy(void); struct prte_rmaps_rf_component_t { prte_rmaps_base_component_t super; char *slot_list; }; typedef struct prte_rmaps_rf_component_t prte_rmaps_rf_component_t; PRTE_MODULE_EXPORT extern prte_rmaps_rf_component_t prte_mca_rmaps_rank_file_component; extern prte_rmaps_base_module_t prte_rmaps_rank_file_module; typedef struct cpu_package_t cpu_package_t; struct prte_rmaps_rank_file_map_t { pmix_object_t super; char *node_name; char slot_list[RMAPS_RANK_FILE_MAX_SLOTS]; }; typedef struct prte_rmaps_rank_file_map_t prte_rmaps_rank_file_map_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_rmaps_rank_file_map_t); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/rank_file/rmaps_rank_file.c0000664000175000017500000006440215203542132023550 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2022 IBM Corporation. All rights reserved. * * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_fd.h" #include "src/util/pmix_if.h" #include "src/util/pmix_net.h" #include "src/util/proc_info.h" #include "src/util/pmix_string_copy.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" #include "src/mca/rmaps/rank_file/rmaps_rank_file.h" #include "src/mca/rmaps/rank_file/rmaps_rank_file_lex.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_show_help.h" static int prte_rmaps_rf_map(prte_job_t *jdata, prte_rmaps_options_t *options); prte_rmaps_base_module_t prte_rmaps_rank_file_module = { .map_job = prte_rmaps_rf_map }; static int prte_rmaps_rank_file_parse(const char *); static char *prte_rmaps_rank_file_parse_string_or_int(void); static int prte_rmaps_rf_lsf_convert_affinity_to_rankfile(char *affinity_file, char **aff_rankfile); static int prte_rmaps_rf_process_lsf_affinity_hostfile(prte_job_t *jdata, prte_rmaps_options_t *options, char *affinity_file); /* * Local variable */ static pmix_pointer_array_t rankmap; static int num_ranks = 0; /* * Create a rank_file mapping for the job. */ static int prte_rmaps_rf_map(prte_job_t *jdata, prte_rmaps_options_t *options) { prte_app_context_t *app = NULL; int32_t i, k; pmix_list_t node_list; prte_node_t *node, *nd, *root_node; pmix_rank_t rank, vpid_start; int32_t num_slots; prte_rmaps_rank_file_map_t *rfmap; int32_t relative_index, tmp_cnt; int rc; prte_proc_t *proc; pmix_mca_base_component_t *c = &prte_mca_rmaps_rank_file_component.super; char *slots = NULL; bool initial_map = true; char *rankfile = NULL; char *affinity_file = NULL; hwloc_cpuset_t proc_bitmap, bitmap; char *cpu_bitmap; char *avail_bitmap = NULL; char *overlap_bitmap = NULL; bool physical; /* only handle initial launch of rf job */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rf: job %s being restarted - rank_file cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (NULL != jdata->map->req_mapper && 0 != strcasecmp(jdata->map->req_mapper, c->pmix_mca_component_name)) { /* a mapper has been specified, and it isn't me */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rf: job %s not using rank_file mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (PRTE_MAPPING_BYUSER != PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) { /* NOT FOR US */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rf: job %s not using rankfile policy", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (options->ordered) { /* NOT FOR US */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rf: job %s binding order requested - rank_file cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, (void **) &rankfile, PMIX_STRING) || NULL == rankfile) { /* we cannot do it */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rf: job %s no rankfile specified", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_BAD_PARAM; } physical = prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_PHYSICAL_CPUS, NULL, PMIX_BOOL); pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rank_file: mapping job %s", PRTE_JOBID_PRINT(jdata->nspace)); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->pmix_mca_component_name); options->map = PRTE_MAPPING_BYUSER; /* setup the node list */ PMIX_CONSTRUCT(&node_list, pmix_list_t); /* pickup the first app - there must be at least one */ app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, 0); if (NULL == app) { rc = PRTE_ERR_SILENT; goto error; } /* start at the beginning... */ vpid_start = 0; jdata->num_procs = 0; PMIX_CONSTRUCT(&rankmap, pmix_pointer_array_t); rc = pmix_pointer_array_init(&rankmap, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != rc) { PMIX_DESTRUCT(&rankmap); return PRTE_ERROR; } /* parse the rankfile, storing its results in the rankmap */ if (PRTE_SUCCESS != (rc = prte_rmaps_rank_file_parse(rankfile))) { rc = PRTE_ERR_SILENT; goto error; } /* cycle through the app_contexts, mapping them sequentially */ for (i = 0; i < jdata->apps->size; i++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } /* for each app_context, we have to get the list of nodes that it can * use since that can now be modified with a hostfile and/or -host * option */ rc = prte_rmaps_base_get_target_nodes(&node_list, &num_slots, jdata, app, options->map, initial_map, false, true); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* flag that all subsequent requests should not reset the node->mapped flag */ initial_map = false; /* set the number of procs to the number of entries in that rankfile */ if (PRTE_FLAG_TEST(app, PRTE_APP_FLAG_COMPUTED)) { app->num_procs = num_ranks; if (0 == app->num_procs) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_SILENT; goto error; } } for (k = 0; k < app->num_procs; k++) { rank = vpid_start + k; /* get the rankfile entry for this rank */ rfmap = (prte_rmaps_rank_file_map_t *) pmix_pointer_array_get_item(&rankmap, rank); if (NULL == rfmap) { /* if this job was given a slot-list, then use it */ if (NULL != options->cpuset) { slots = options->cpuset; } else if (NULL != prte_hwloc_default_cpu_list) { /* if we were give a default slot-list, then use it */ slots = prte_hwloc_default_cpu_list; } else { /* all ranks must be specified */ pmix_show_help("help-rmaps_rank_file.txt", "missing-rank", true, rank, rankfile); rc = PRTE_ERR_SILENT; goto error; } /* take the next node off of the available list */ node = NULL; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { /* if adding one to this node would oversubscribe it, then try * the next one */ if (nd->slots <= (int) nd->num_procs) { continue; } /* take this one */ node = nd; break; } if (NULL == node) { /* all would be oversubscribed, so take the least loaded one */ k = (int32_t) UINT32_MAX; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { if (nd->num_procs < (pmix_rank_t) k) { k = nd->num_procs; node = nd; } } } /* if we still have nothing, then something is very wrong */ if (NULL == node) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } } else { if (0 < strlen(rfmap->slot_list)) { slots = rfmap->slot_list; } else { slots = NULL; } /* find the node where this proc was assigned */ node = NULL; PMIX_LIST_FOREACH(nd, &node_list, prte_node_t) { if (NULL != rfmap->node_name && prte_quickmatch(nd, rfmap->node_name) ){ node = nd; break; } else if (NULL != rfmap->node_name && (('+' == rfmap->node_name[0]) && (('n' == rfmap->node_name[1]) || ('N' == rfmap->node_name[1])))) { relative_index = atoi(strtok(rfmap->node_name, "+n")); if (relative_index >= (int) pmix_list_get_size(&node_list) || (0 > relative_index)) { pmix_show_help("help-rmaps_rank_file.txt", "bad-index", true, rfmap->node_name); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); return PRTE_ERR_BAD_PARAM; } root_node = (prte_node_t *) pmix_list_get_first(&node_list); for (tmp_cnt = 0; tmp_cnt < relative_index; tmp_cnt++) { root_node = (prte_node_t *) pmix_list_get_next(root_node); } node = root_node; break; } } } if (NULL == node) { pmix_show_help("help-rmaps_rank_file.txt", "bad-host", true, rfmap->node_name); rc = PRTE_ERR_SILENT; goto error; } if (!options->donotlaunch) { rc = prte_rmaps_base_check_support(jdata, node, options); if (PRTE_SUCCESS != rc) { return rc; } } prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto error; } if (!prte_rmaps_base_check_avail(jdata, app, node, &node_list, NULL, options)) { pmix_show_help("help-rmaps_rank_file.txt", "bad-host", true, rfmap->node_name); rc = PRTE_ERR_SILENT; goto error; } /* check if we are oversubscribed */ rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_SUCCESS != rc) { PMIX_RELEASE(proc); goto error; } proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } /* set the vpid */ proc->name.rank = rank; /* insert the proc into the proper place */ PMIX_RETAIN(proc); rc = pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(proc); goto error; } jdata->num_procs++; PMIX_RELEASE(proc); /* Assign this process to these slots * Allow for overload in cases where different ranks are assigned to * the same PU, but it must be requested by the user. */ if (NULL != slots && (PRTE_BIND_TO_NONE != PRTE_GET_BINDING_POLICY(jdata->map->binding) || options->overload) ) { if (NULL == node->topology || NULL == node->topology->topo) { // Not allowed - for rank-file, we must have the topology pmix_show_help("help-prte-rmaps-base.txt", "rmaps:no-topology", true, node->name); rc = PRTE_ERR_SILENT; goto error; } proc_bitmap = hwloc_bitmap_alloc(); /* parse the slot_list to find the package and core */ rc = prte_hwloc_base_cpu_list_parse(slots, node->topology->topo, options->use_hwthreads, proc_bitmap); if (PRTE_ERR_NOT_FOUND == rc) { char *tmp = prte_hwloc_base_cset2str(hwloc_topology_get_allowed_cpuset(node->topology->topo), false, physical, node->topology->topo); pmix_show_help("help-rmaps_rank_file.txt", "missing-cpu", true, prte_tool_basename, slots, tmp); free(tmp); rc = PRTE_ERR_SILENT; hwloc_bitmap_free(proc_bitmap); goto error; } else if (PRTE_ERROR == rc) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_SILENT; hwloc_bitmap_free(proc_bitmap); goto error; } else if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); hwloc_bitmap_free(proc_bitmap); goto error; } /* note that we cannot set the proc locale to any specific object * as the slot list may have assigned it to more than one - so * leave that field NULL */ /* set the proc to the specified map */ hwloc_bitmap_list_asprintf(&cpu_bitmap, proc_bitmap); proc->cpuset = strdup(cpu_bitmap); pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rank_file: convert slots from <%s> to <%s>", slots, cpu_bitmap); /* Check to see if these slots are available on this node */ if (!hwloc_bitmap_isincluded(proc_bitmap, node->available) && !options->overload) { bitmap = hwloc_bitmap_alloc(); hwloc_bitmap_list_asprintf(&avail_bitmap, node->available); hwloc_bitmap_andnot(bitmap, proc_bitmap, node->available); hwloc_bitmap_list_asprintf(&overlap_bitmap, bitmap); pmix_show_help("help-rmaps_rank_file.txt", "rmaps:proc-slots-overloaded", true, PRTE_NAME_PRINT(&proc->name), node->name, proc->cpuset, avail_bitmap, overlap_bitmap); hwloc_bitmap_free(bitmap); hwloc_bitmap_free(proc_bitmap); rc = PRTE_ERR_OUT_OF_RESOURCE; goto error; } /* Mark these slots as taken on this node */ hwloc_bitmap_andnot(node->available, node->available, proc_bitmap); /* cleanup */ free(cpu_bitmap); hwloc_bitmap_free(proc_bitmap); } } /* update the starting point */ vpid_start += app->num_procs; /* cleanup the node list - it can differ from one app_context * to another, so we have to get it every time */ PMIX_LIST_DESTRUCT(&node_list); PMIX_CONSTRUCT(&node_list, pmix_list_t); } PMIX_LIST_DESTRUCT(&node_list); /* cleanup the rankmap */ for (i = 0; i < rankmap.size; i++) { if (NULL != (rfmap = pmix_pointer_array_get_item(&rankmap, i))) { PMIX_RELEASE(rfmap); } } PMIX_DESTRUCT(&rankmap); if (NULL != rankfile) { free(rankfile); } /* compute local/app ranks */ rc = prte_rmaps_base_compute_vpids(jdata, options); return rc; error: PMIX_LIST_DESTRUCT(&node_list); if (NULL != rankfile) { free(rankfile); } return rc; } static int prte_rmaps_rank_file_parse(const char *rankfile) { int token; int rc = PRTE_SUCCESS; int cnt; char *node_name = NULL; char **argv; char buff[RMAPS_RANK_FILE_MAX_SLOTS]; char *value; int rank = -1; int i; prte_node_t *hnp_node; prte_rmaps_rank_file_map_t *rfmap = NULL; pmix_pointer_array_t *assigned_ranks_array; char tmp_rank_assignment[RMAPS_RANK_FILE_MAX_SLOTS]; /* keep track of rank assignments */ assigned_ranks_array = PMIX_NEW(pmix_pointer_array_t); rc = pmix_pointer_array_init(assigned_ranks_array, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(assigned_ranks_array); return PRTE_ERROR; } /* get the hnp node's info */ hnp_node = (prte_node_t *) (prte_node_pool->addr[0]); prte_rmaps_rank_file_done = false; prte_rmaps_rank_file_in = fopen(rankfile, "r"); if (NULL == prte_rmaps_rank_file_in) { pmix_show_help("help-rmaps_rank_file.txt", "no-rankfile", true, prte_tool_basename, rankfile, prte_tool_basename); rc = PRTE_ERR_NOT_FOUND; goto unlock; } while (!prte_rmaps_rank_file_done) { token = prte_rmaps_rank_file_lex(); switch (token) { case PRTE_RANKFILE_ERROR: pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; case PRTE_RANKFILE_QUOTED_STRING: pmix_show_help("help-rmaps_rank_file.txt", "not-supported-rankfile", true, "QUOTED_STRING", rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; case PRTE_RANKFILE_NEWLINE: rank = -1; if (NULL != node_name) { free(node_name); } node_name = NULL; rfmap = NULL; break; case PRTE_RANKFILE_RANK: token = prte_rmaps_rank_file_lex(); if (PRTE_RANKFILE_INT == token) { rank = prte_rmaps_rank_file_value.ival; rfmap = PMIX_NEW(prte_rmaps_rank_file_map_t); pmix_pointer_array_set_item(&rankmap, rank, rfmap); num_ranks++; // keep track of number of provided ranks } else { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; } break; case PRTE_RANKFILE_USERNAME: pmix_show_help("help-rmaps_rank_file.txt", "not-supported-rankfile", true, "USERNAME", rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; case PRTE_RANKFILE_EQUAL: if (rank < 0) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; } token = prte_rmaps_rank_file_lex(); switch (token) { case PRTE_RANKFILE_HOSTNAME: case PRTE_RANKFILE_IPV4: case PRTE_RANKFILE_IPV6: case PRTE_RANKFILE_STRING: case PRTE_RANKFILE_INT: case PRTE_RANKFILE_RELATIVE: if (PRTE_RANKFILE_INT == token) { snprintf(buff,RMAPS_RANK_FILE_MAX_SLOTS, "%d", prte_rmaps_rank_file_value.ival); value = buff; } else { value = prte_rmaps_rank_file_value.sval; } argv = PMIx_Argv_split(value, '@'); cnt = PMIx_Argv_count(argv); if (NULL != node_name) { free(node_name); } if (1 == cnt) { node_name = strdup(argv[0]); } else if (2 == cnt) { node_name = strdup(argv[1]); } else { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); PMIx_Argv_free(argv); node_name = NULL; goto unlock; } PMIx_Argv_free(argv); // Strip off the FQDN if present, ignore IP addresses if (!prte_keep_fqdn_hostnames && !pmix_net_isaddr(node_name)) { char *ptr; if (NULL != (ptr = strchr(node_name, '.'))) { *ptr = '\0'; } } /* check the rank item */ if (NULL == rfmap) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; } /* check if this is the local node */ if (prte_check_host_is_local(node_name)) { rfmap->node_name = strdup(hnp_node->name); } else { rfmap->node_name = strdup(node_name); } } break; case PRTE_RANKFILE_SLOT: if (NULL == node_name || rank < 0 || NULL == (value = prte_rmaps_rank_file_parse_string_or_int())) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); goto unlock; } /* check for a duplicate rank assignment */ if (NULL != pmix_pointer_array_get_item(assigned_ranks_array, rank)) { pmix_show_help("help-rmaps_rank_file.txt", "bad-assign", true, rank, pmix_pointer_array_get_item(assigned_ranks_array, rank), rankfile); rc = PRTE_ERR_BAD_PARAM; free(value); goto unlock; } else { /* prepare rank assignment string for the help message in case of a bad-assign */ snprintf(tmp_rank_assignment, RMAPS_RANK_FILE_MAX_SLOTS, "%s slot=%s", node_name, value); pmix_pointer_array_set_item(assigned_ranks_array, 0, tmp_rank_assignment); } /* check the rank item */ if (NULL == rfmap) { pmix_show_help("help-rmaps_rank_file.txt", "bad-syntax", true, rankfile); rc = PRTE_ERR_BAD_PARAM; PRTE_ERROR_LOG(rc); free(value); goto unlock; } for (i = 0; i < RMAPS_RANK_FILE_MAX_SLOTS && '\0' != value[i]; i++) { rfmap->slot_list[i] = value[i]; } free(value); break; } } fclose(prte_rmaps_rank_file_in); prte_rmaps_rank_file_lex_destroy(); unlock: if (NULL != node_name) { free(node_name); } PMIX_RELEASE(assigned_ranks_array); return rc; } static char *prte_rmaps_rank_file_parse_string_or_int(void) { int rc; char tmp_str[RMAPS_RANK_FILE_MAX_SLOTS]; if (PRTE_RANKFILE_EQUAL != prte_rmaps_rank_file_lex()) { return NULL; } rc = prte_rmaps_rank_file_lex(); switch (rc) { case PRTE_RANKFILE_STRING: return strdup(prte_rmaps_rank_file_value.sval); case PRTE_RANKFILE_INT: snprintf(tmp_str, RMAPS_RANK_FILE_MAX_SLOTS, "%d", prte_rmaps_rank_file_value.ival); return strdup(tmp_str); default: return NULL; } } prrte-4.1.1~rc2/src/mca/rmaps/round_robin/0000775000175000017500000000000015203542132020630 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/rmaps/round_robin/rmaps_rr.c0000664000175000017500000001534315203542132022627 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/mca/errmgr/errmgr.h" #include "src/util/error_strings.h" #include "src/util/pmix_show_help.h" #include "rmaps_rr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" /* * Create a round-robin mapping for the job. */ static int prte_rmaps_rr_map(prte_job_t *jdata, prte_rmaps_options_t *options) { prte_app_context_t *app; int i; pmix_list_t node_list; int32_t num_slots; int rc; pmix_mca_base_component_t *c = &prte_mca_rmaps_round_robin_component; bool initial_map = true; /* this mapper can only handle initial launch * when rr mapping is desired - allow * restarting of failed apps */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: job %s is being restarted - rr cannot map", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (NULL != jdata->map->req_mapper && 0 != strcasecmp(jdata->map->req_mapper, c->pmix_mca_component_name)) { /* a mapper has been specified, and it isn't me */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: job %s not using rr mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } if (PRTE_MAPPING_RR < PRTE_GET_MAPPING_POLICY(jdata->map->mapping)) { /* I don't know how to do these - defer */ pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: job %s not using rr mapper", PRTE_JOBID_PRINT(jdata->nspace)); return PRTE_ERR_TAKE_NEXT_OPTION; } pmix_output_verbose(5, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: mapping job %s", PRTE_JOBID_PRINT(jdata->nspace)); /* flag that I did the mapping */ if (NULL != jdata->map->last_mapper) { free(jdata->map->last_mapper); } jdata->map->last_mapper = strdup(c->pmix_mca_component_name); /* start at the beginning... */ jdata->num_procs = 0; /* cycle through the app_contexts, mapping them sequentially */ for (i = 0; i < jdata->apps->size; i++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i); if (NULL == app) { continue; } /* setup the nodelist here in case we jump to error */ PMIX_CONSTRUCT(&node_list, pmix_list_t); /* for each app_context, we have to get the list of nodes that it can * use since that can now be modified with a hostfile and/or -host * option */ rc = prte_rmaps_base_get_target_nodes(&node_list, &num_slots, jdata, app, jdata->map->mapping, initial_map, false, false); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* flag that all subsequent requests should not reset the node->mapped flag */ initial_map = false; /* Make assignments */ if (PRTE_MAPPING_BYNODE == options->map) { rc = prte_rmaps_rr_bynode(jdata, app, &node_list, num_slots, app->num_procs, options); } else if (PRTE_MAPPING_BYSLOT == options->map) { rc = prte_rmaps_rr_byslot(jdata, app, &node_list, num_slots, app->num_procs, options); } else if (PRTE_MAPPING_PELIST == options->map) { rc = prte_rmaps_rr_bycpu(jdata, app, &node_list, num_slots, app->num_procs, options); } else { rc = prte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, options); if (PRTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so */ PRTE_SET_MAPPING_POLICY(jdata->map->mapping, PRTE_MAPPING_BYSLOT); options->map = PRTE_MAPPING_BYSLOT; rc = prte_rmaps_rr_byslot(jdata, app, &node_list, num_slots, app->num_procs, options); } } if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto error; } /* track the total number of processes we mapped - must update * this value AFTER we compute vpids so that computation * is done correctly */ jdata->num_procs += app->num_procs; /* cleanup the node list - it can differ from one app_context * to another, so we have to get it every time */ PMIX_LIST_DESTRUCT(&node_list); } /* calculate the ranks for this job */ rc = prte_rmaps_base_compute_vpids(jdata, options); return rc; error: PMIX_LIST_DESTRUCT(&node_list); return rc; } prte_rmaps_base_module_t prte_rmaps_round_robin_module = { .map_job = prte_rmaps_rr_map }; prrte-4.1.1~rc2/src/mca/rmaps/round_robin/owner.txt0000664000175000017500000000024415203542132022523 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/rmaps/round_robin/rmaps_rr.h0000664000175000017500000000520615203542132022631 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Mapping */ #ifndef PRTE_RMAPS_RR_H #define PRTE_RMAPS_RR_H #include "prte_config.h" #include "src/class/pmix_list.h" #include "src/hwloc/hwloc-internal.h" #include "src/mca/rmaps/rmaps.h" BEGIN_C_DECLS PRTE_MODULE_EXPORT extern prte_rmaps_base_component_t prte_mca_rmaps_round_robin_component; extern prte_rmaps_base_module_t prte_rmaps_round_robin_module; PRTE_MODULE_EXPORT int prte_rmaps_rr_bynode(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t nprocs, prte_rmaps_options_t *options); PRTE_MODULE_EXPORT int prte_rmaps_rr_byslot(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t nprocs, prte_rmaps_options_t *options); PRTE_MODULE_EXPORT int prte_rmaps_rr_byobj(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options); PRTE_MODULE_EXPORT int prte_rmaps_rr_bycpu(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/rmaps/round_robin/rmaps_rr_component.c0000664000175000017500000000641015203542132024704 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "rmaps_rr.h" #include "src/mca/rmaps/base/rmaps_private.h" /* * Local functions */ static int prte_rmaps_round_robin_register(void); static int prte_rmaps_round_robin_open(void); static int prte_rmaps_round_robin_close(void); static int prte_rmaps_round_robin_query(pmix_mca_base_module_t **module, int *priority); static int my_priority; prte_rmaps_base_component_t prte_mca_rmaps_round_robin_component = { PRTE_RMAPS_BASE_VERSION_4_0_0, .pmix_mca_component_name = "round_robin", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_open_component = prte_rmaps_round_robin_open, .pmix_mca_close_component = prte_rmaps_round_robin_close, .pmix_mca_query_component = prte_rmaps_round_robin_query, .pmix_mca_register_component_params = prte_rmaps_round_robin_register, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, rmaps, round_robin) /** * component register/open/close/init function */ static int prte_rmaps_round_robin_register(void) { my_priority = 10; (void) pmix_mca_base_component_var_register(&prte_mca_rmaps_round_robin_component, "priority", "Priority of the rr rmaps component", PMIX_MCA_BASE_VAR_TYPE_INT, &my_priority); return PRTE_SUCCESS; } static int prte_rmaps_round_robin_open(void) { return PRTE_SUCCESS; } static int prte_rmaps_round_robin_query(pmix_mca_base_module_t **module, int *priority) { /* the RMAPS framework is -only- opened on HNP's, * so no need to check for that here */ *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_rmaps_round_robin_module; return PRTE_SUCCESS; } /** * Close all subsystems. */ static int prte_rmaps_round_robin_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/rmaps/round_robin/rmaps_rr_mappers.c0000664000175000017500000007250415203542132024360 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2021 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "rmaps_rr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/base/rmaps_private.h" int prte_rmaps_rr_byslot(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options) { int i, rc=PRTE_SUCCESS, nprocs_mapped, ncpus; prte_node_t *node, *nd; int extra_procs_to_assign = 0, nxtra_nodes = 0; float balance; prte_proc_t *proc; bool second_pass = false; prte_binding_policy_t savebind = options->bind; pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: mapping by slot for job %s slots %d num_procs %lu", PRTE_JOBID_PRINT(jdata->nspace), (int) num_slots, (unsigned long) num_procs); /* check to see if we can map all the procs */ if (num_slots < (int) app->num_procs) { if (!options->oversubscribe) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } else { if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { jdata->map->binding = PRTE_BIND_TO_NONE; options->bind = PRTE_BIND_TO_NONE; savebind = options->bind; } } } nprocs_mapped = 0; pass: PMIX_LIST_FOREACH_SAFE(node, nd, node_list, prte_node_t) { pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:slot working node %s", node->name); prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto errout; } /* compute the number of procs to go on this node */ if (second_pass) { options->nprocs = extra_procs_to_assign; if (0 < nxtra_nodes) { --nxtra_nodes; if (0 == nxtra_nodes) { --extra_procs_to_assign; } } } else { if (!options->donotlaunch) { rc = prte_rmaps_base_check_support(jdata, node, options); if (PRTE_SUCCESS != rc) { return rc; } } /* assign a number of procs equal to the number of available slots */ if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { options->nprocs = node->slots_available; } else { options->nprocs = node->slots; } } if (!options->oversubscribe) { /* since oversubscribe is not allowed, cap our usage * at the number of available slots. */ if (node->slots_available < options->nprocs) { options->nprocs = node->slots_available; } } /* if the number of procs is greater than the number of CPUs * on this node, but less or equal to the number of slots, * then we are not oversubscribed but we are overloaded. If * the user didn't specify a required binding, then we set * the binding policy to do-not-bind for this node */ ncpus = prte_rmaps_base_get_ncpus(node, NULL, options); if (options->nprocs > ncpus && options->nprocs <= node->slots_available && !PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { options->bind = PRTE_BIND_TO_NONE; jdata->map->binding = PRTE_BIND_TO_NONE; } if (!prte_rmaps_base_check_avail(jdata, app, node, node_list, NULL, options)) { rc = PRTE_ERR_OUT_OF_RESOURCE; options->bind = savebind; continue; } pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:slot assigning %d procs to node %s", (int) options->nprocs, node->name); for (i = 0; i < options->nprocs && nprocs_mapped < app->num_procs; i++) { proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options); if (NULL == proc) { /* move on to the next node */ rc = PRTE_ERR_SILENT; break; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); goto errout; } PMIX_RELEASE(proc); } if (nprocs_mapped == app->num_procs) { return PRTE_SUCCESS; } options->bind = savebind; if(NULL != options->target) { hwloc_bitmap_free(options->target); options->target = NULL; } } if (second_pass) { errout: if (PRTE_ERR_SILENT != rc) { pmix_show_help("help-prte-rmaps-base.txt", "failed-map", true, PRTE_ERROR_NAME(rc), (NULL == app) ? "N/A" : app->app, (NULL == app) ? -1 : app->num_procs, prte_rmaps_base_print_mapping(options->map), prte_hwloc_base_print_binding(options->bind)); } return PRTE_ERR_SILENT; } pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:slot job %s is oversubscribed - performing second pass", PRTE_JOBID_PRINT(jdata->nspace)); /* second pass: if we haven't mapped everyone yet, it is * because we are oversubscribed. All of the nodes that are * at max_slots have been removed from the list as that specifies * a hard boundary, so the nodes remaining are available for * handling the oversubscription. Figure out how many procs * to add to each of them. */ balance = (float) ((int) app->num_procs - nprocs_mapped) / (float) pmix_list_get_size(node_list); extra_procs_to_assign = (int) balance; if (0 < (balance - (float) extra_procs_to_assign)) { /* compute how many nodes need an extra proc */ nxtra_nodes = app->num_procs - nprocs_mapped - (extra_procs_to_assign * pmix_list_get_size(node_list)); /* add one so that we add an extra proc to the first nodes * until all procs are mapped */ extra_procs_to_assign++; } // Rescan the nodes second_pass = true; goto pass; } int prte_rmaps_rr_bynode(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options) { int rc=PRTE_SUCCESS, j, nprocs_mapped, ncpus; prte_node_t *node, *nd; bool second_pass = false; prte_proc_t *proc; prte_binding_policy_t savebind = options->bind; pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: mapping by node for job %s app %d slots %d num_procs %lu", PRTE_JOBID_PRINT(jdata->nspace), (int) app->idx, (int) num_slots, (unsigned long) num_procs); /* quick check to see if we can map all the procs */ if (num_slots < (int) app->num_procs) { if (!options->oversubscribe) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } else { if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { jdata->map->binding = PRTE_BIND_TO_NONE; options->bind = PRTE_BIND_TO_NONE; savebind = PRTE_BIND_TO_NONE; } } } nprocs_mapped = 0; pass: /* divide the procs evenly across all nodes - this is the * average we have to maintain as we go, but we adjust * the number on each node to reflect its available slots. * Obviously, if all nodes have the same number of slots, * then the avg is what we get on each node - this is * the most common situation. */ options->nprocs = (app->num_procs - nprocs_mapped) / pmix_list_get_size(node_list); if (0 == options->nprocs) { /* if there are less procs than nodes, we have to * place at least one/node */ options->nprocs = 1; } PMIX_LIST_FOREACH_SAFE(node, nd, node_list, prte_node_t) { prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto errout; } if (!options->oversubscribe) { /* since oversubscribe is not allowed, cap our usage * at the number of available slots. */ if (node->slots_available < options->nprocs) { options->nprocs = node->slots_available; } } /* if the number of procs is greater than the number of CPUs * on this node, but less or equal to the number of slots, * then we are not oversubscribed but we are overloaded. If * the user didn't specify a required binding, then we set * the binding policy to do-not-bind for this node */ ncpus = prte_rmaps_base_get_ncpus(node, NULL, options); if (options->nprocs > ncpus && options->nprocs <= node->slots_available && !PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { options->bind = PRTE_BIND_TO_NONE; jdata->map->binding = PRTE_BIND_TO_NONE; } if (!prte_rmaps_base_check_avail(jdata, app, node, node_list, NULL, options)) { rc = PRTE_ERR_OUT_OF_RESOURCE; options->bind = savebind; continue; } PMIX_OUTPUT_VERBOSE((10, prte_rmaps_base_framework.framework_output, "%s NODE %s ASSIGNING %d PROCS", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, options->nprocs)); for (j=0; j < options->nprocs && nprocs_mapped < app->num_procs; j++) { proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options); if (NULL == proc) { /* move to next node */ rc = PRTE_ERR_SILENT; break; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); goto errout; } PMIX_RELEASE(proc); } if (nprocs_mapped == app->num_procs) { return PRTE_SUCCESS; } options->bind = savebind; if(NULL != options->target) { hwloc_bitmap_free(options->target); options->target = NULL; } } if (second_pass) { errout: /* unable to do it */ if (PRTE_ERR_SILENT != rc) { pmix_show_help("help-prte-rmaps-base.txt", "failed-map", true, PRTE_ERROR_NAME(rc), (NULL == app) ? "N/A" : app->app, (NULL == app) ? -1 : app->num_procs, prte_rmaps_base_print_mapping(options->map), prte_hwloc_base_print_binding(options->bind)); } return PRTE_ERR_SILENT; } pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:node job %s is oversubscribed - performing second pass", PRTE_JOBID_PRINT(jdata->nspace)); /* second pass: if we haven't mapped everyone yet, it is * because we are oversubscribed. All of the nodes that are * at max_slots have been removed from the list as that specifies * a hard boundary, so the nodes remaining are available for * handling the oversubscription. */ second_pass = true; goto pass; } /* mapping by cpu */ int prte_rmaps_rr_bycpu(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options) { int i, rc, nprocs_mapped, ncpus; prte_node_t *node, *nd; prte_proc_t *proc; char **tmp; int ntomap; bool second_pass = false; int extra_procs_to_assign = 0, nxtra_nodes = 0; float balance; char *savecpuset = NULL; prte_binding_policy_t savebind = options->bind; PRTE_HIDE_UNUSED_PARAMS(num_procs); pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: mapping by cpu for job %s slots %d num_procs %lu", PRTE_JOBID_PRINT(jdata->nspace), (int) num_slots, (unsigned long)app->num_procs); /* check to see if we can map all the procs */ if (num_slots < (int) app->num_procs) { if (!options->oversubscribe) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } else { if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { jdata->map->binding = PRTE_BIND_TO_NONE; options->bind = PRTE_BIND_TO_NONE; savebind = PRTE_BIND_TO_NONE; } } } nprocs_mapped = 0; savecpuset = strdup(options->cpuset); pass: PMIX_LIST_FOREACH_SAFE(node, nd, node_list, prte_node_t) { pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:cpu working node %s", node->name); // get the cpuset and check that all specified PEs are available // on this node prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto errout; } if (second_pass) { options->nprocs = extra_procs_to_assign; if (0 < nxtra_nodes) { --nxtra_nodes; if (0 == nxtra_nodes) { --extra_procs_to_assign; } } } else if (options->ordered || !options->overload) { // see how many PEs we were given tmp = PMIx_Argv_split(options->cpuset, ','); ntomap = PMIx_Argv_count(tmp); PMIx_Argv_free(tmp); options->nprocs = ntomap; } else { /* assign a number of procs equal to the number of available slots */ if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL)) { options->nprocs = node->slots_available; } else { options->nprocs = node->slots; } } if (!options->oversubscribe) { /* oversubscribe is not allowed, so cap our usage * at the number of available slots. */ if (node->slots_available < options->nprocs) { options->nprocs = node->slots_available; } } /* if the number of procs is greater than the number of CPUs * on this node, but less or equal to the number of slots, * then we are not oversubscribed but we are overloaded. If * the user didn't specify a required binding, then we set * the binding policy to do-not-bind for this node */ ncpus = prte_rmaps_base_get_ncpus(node, NULL, options); if (options->nprocs > ncpus && options->nprocs <= node->slots_available && !PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { options->bind = PRTE_BIND_TO_NONE; jdata->map->binding = PRTE_BIND_TO_NONE; } if (!prte_rmaps_base_check_avail(jdata, app, node, node_list, NULL, options)) { rc = PRTE_ERR_OUT_OF_RESOURCE; options->bind = savebind; continue; } pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:cpu assigning %d procs to node %s", (int) options->nprocs, node->name); for (i = 0; i < options->nprocs && nprocs_mapped < app->num_procs; i++) { proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options); if (NULL == proc) { rc = PRTE_ERR_SILENT; goto errout; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); goto errout; } PMIX_RELEASE(proc); } if (nprocs_mapped == app->num_procs) { if (NULL != options->target) { hwloc_bitmap_free(options->target); options->target = NULL; } if (NULL != options->job_cpuset) { hwloc_bitmap_free(options->job_cpuset); options->job_cpuset = NULL; } options->cpuset = strdup(savecpuset); if (NULL != savecpuset) { free(savecpuset); } return PRTE_SUCCESS; } if (NULL != options->target) { hwloc_bitmap_free(options->target); options->target = NULL; } if (NULL != options->job_cpuset) { hwloc_bitmap_free(options->job_cpuset); options->job_cpuset = NULL; } if (NULL != options->cpuset) { free(options->cpuset); } options->cpuset = strdup(savecpuset); } // next node /* second pass: if we haven't mapped everyone yet, it is * because we are oversubscribed. All of the nodes that are * at max_slots have been removed from the list as that specifies * a hard boundary, so the nodes remaining are available for * handling the oversubscription. Figure out how many procs * to add to each of them. */ if (options->oversubscribe && !second_pass) { balance = (float) ((int) app->num_procs - nprocs_mapped) / (float) pmix_list_get_size(node_list); extra_procs_to_assign = (int) balance; if (0 < (balance - (float) extra_procs_to_assign)) { /* compute how many nodes need an extra proc */ nxtra_nodes = app->num_procs - nprocs_mapped - (extra_procs_to_assign * pmix_list_get_size(node_list)); /* add one so that we add an extra proc to the first nodes * until all procs are mapped */ extra_procs_to_assign++; } /* restore the cpuset */ if (NULL != options->cpuset) { free(options->cpuset); } options->cpuset = strdup(savecpuset); // Rescan the nodes second_pass = true; goto pass; } errout: /* if we get here, then we were unable to map all the procs */ if (PRTE_ERR_SILENT != rc) { pmix_show_help("help-prte-rmaps-rr.txt", "prte-rmaps-rr:not-enough-cpus", true, app->app, app->num_procs, savecpuset); } if (NULL != savecpuset) { free(savecpuset); } return PRTE_ERR_SILENT; } /* mapping by hwloc object looks a lot like mapping by node, * but has the added complication of possibly having different * numbers of objects on each node */ int prte_rmaps_rr_byobj(prte_job_t *jdata, prte_app_context_t *app, pmix_list_t *node_list, int32_t num_slots, pmix_rank_t num_procs, prte_rmaps_options_t *options) { int rc=PRTE_SUCCESS, nprocs_mapped; prte_node_t *node, *nnext; int ncpus; prte_proc_t *proc; bool nodefull, allfull, outofcpus=false; hwloc_obj_t obj = NULL; unsigned j, nobjs; pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr:byobj mapping by %s for job %s slots %d num_procs %lu", hwloc_obj_type_string(options->maptype), PRTE_JOBID_PRINT(jdata->nspace), (int) num_slots, (unsigned long) num_procs); /* quick check to see if we can map all the procs */ if (num_slots < app->num_procs) { if (!options->oversubscribe) { pmix_show_help("help-prte-rmaps-base.txt", "prte-rmaps-base:alloc-error", true, app->num_procs, app->app, prte_process_info.nodename); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); return PRTE_ERR_SILENT; } else { if (!PRTE_BINDING_POLICY_IS_SET(jdata->map->binding)) { jdata->map->binding = PRTE_BIND_TO_NONE; options->bind = PRTE_BIND_TO_NONE; } } } /* there are two modes for mapping by object: span and not-span. The * span mode essentially operates as if there was just a single * "super-node" in the system - i.e., it balances the load across * all objects of the indicated type regardless of their location. * In essence, it acts as if we placed one proc on each object, cycling * across all objects on all nodes, and then wrapped around to place * another proc on each object, doing so until all procs were placed. * * In contrast, the non-span mode operates similar to byslot mapping. * All slots on each node are filled, assigning each proc to an object * on that node in a balanced fashion, and then the mapper moves on * to the next node. Thus, procs tend to be "front loaded" onto the * list of nodes, as opposed to being "load balanced" in the span mode */ allfull = true; nprocs_mapped = 0; do { allfull = true; PMIX_LIST_FOREACH_SAFE(node, nnext, node_list, prte_node_t) { outofcpus = false; prte_rmaps_base_get_cpuset(jdata, node, options); if (NULL == options->job_cpuset) { // the prior function will have printed out the error rc = PRTE_ERR_SILENT; goto errout; } if (!options->donotlaunch) { rc = prte_rmaps_base_check_support(jdata, node, options); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); goto errout; } } options->nobjs = 0; /* have to delay checking for availability until we have the object */ /* get the number of objects of this type on this node */ nobjs = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, options->maptype); if (0 == nobjs) { /* this node doesn't have any objects of this type, so * we might as well drop it from consideration */ pmix_list_remove_item(node_list, &node->super); PMIX_RELEASE(node); continue; } pmix_output_verbose(2, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: found %u %s objects on node %s", nobjs, hwloc_obj_type_string(options->maptype), node->name); nodefull = false; redo: for (j=0; j < nobjs && nprocs_mapped < app->num_procs && !nodefull; j++) { pmix_output_verbose(10, prte_rmaps_base_framework.framework_output, "mca:rmaps:rr: assigning proc to object %d", j); /* get the hwloc object */ obj = prte_hwloc_base_get_obj_by_type(node->topology->topo, options->maptype, j); if (NULL == obj) { /* out of objects on this node */ break; } /* does this object have enough available cpus to * support the requested cpus_per_rank? */ ncpus = prte_rmaps_base_get_ncpus(node, obj, options); if (ncpus < options->cpus_per_rank && !options->overload) { outofcpus = true; continue; } options->nprocs = 1; if (!prte_rmaps_base_check_avail(jdata, app, node, node_list, obj, options)) { rc = PRTE_ERR_OUT_OF_RESOURCE; PRTE_ERROR_LOG(rc); // out of resources on this node break; } proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, obj, options); if (NULL == proc) { rc = PRTE_ERR_OUT_OF_RESOURCE; goto errout; } nprocs_mapped++; rc = prte_rmaps_base_check_oversubscribed(jdata, app, node, options); if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* move to next node */ pmix_list_remove_item(node_list, &node->super); PMIX_RELEASE(node); nodefull = true; PMIX_RELEASE(proc); break; } else if (PRTE_SUCCESS != rc) { /* got an error */ PMIX_RELEASE(proc); goto errout; } PMIX_RELEASE(proc); allfull = false; } if (nprocs_mapped < app->num_procs && !allfull && !nodefull && !outofcpus && !options->mapspan) { // keep working these objects until full goto redo; } // move to the next node if (NULL != options->target) { hwloc_bitmap_free(options->target); options->target = NULL; } } } while (nprocs_mapped < app->num_procs && !allfull); if (nprocs_mapped == app->num_procs) { return PRTE_SUCCESS; } errout: if (PRTE_ERR_SILENT == rc) { return rc; } if (outofcpus) { /* ran out of cpus */ pmix_show_help("help-prte-rmaps-base.txt", "allocation-overload", true, app->app, app->num_procs, prte_rmaps_base_print_mapping(options->map), prte_hwloc_base_print_binding(options->bind)); return PRTE_ERR_SILENT; } pmix_show_help("help-prte-rmaps-base.txt", "failed-map", true, PRTE_ERROR_NAME(rc), app->app, app->num_procs, prte_rmaps_base_print_mapping(options->map), prte_hwloc_base_print_binding(options->bind)); return PRTE_ERR_SILENT; } prrte-4.1.1~rc2/src/mca/rmaps/round_robin/Makefile.am0000664000175000017500000000355015203542132022667 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-prte-rmaps-rr.txt sources = \ rmaps_rr.c \ rmaps_rr.h \ rmaps_rr_component.c \ rmaps_rr_mappers.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_rmaps_round_robin_DSO component_noinst = component_install = prte_mca_rmaps_round_robin.la else component_noinst = libprtemca_rmaps_round_robin.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_rmaps_round_robin_la_SOURCES = $(sources) prte_mca_rmaps_round_robin_la_LDFLAGS = -module -avoid-version prte_mca_rmaps_round_robin_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_rmaps_round_robin_la_SOURCES =$(sources) libprtemca_rmaps_round_robin_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/rmaps/round_robin/help-prte-rmaps-rr.txt0000664000175000017500000000427015203542132025035 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2018-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for PRTE's prun. # [prte-rmaps-rr:not-enough-cpus] There are not enough slots available in the system or not enough CPUs in the specified PE-LIST to map the number of processes requested by the application: app: %s Number of procs: %d pe-list: %s Either request fewer procs for your application, make more slots available for use, or expand the pe-list. A "slot" is the PRRTE term for an allocatable unit where we can launch a process. The number of slots available are defined by the environment in which PRRTE processes are run: 1. Hostfile, via "slots=N" clauses (N defaults to number of processor cores if not provided) 2. The --host command line parameter, via a ":N" suffix on the hostname (N defaults to 1 if not provided) 3. Resource manager (e.g., SLURM, PBS/Torque, LSF, etc.) 4. If none of a hostfile, the --host command line parameter, or an RM is present, PRRTE defaults to the number of processor cores In all the above cases, if you want PRRTE to default to the number of hardware threads instead of the number of processor cores, use the --use-hwthread-cpus option. Alternatively, you can use the --map-by :OVERSUBSCRIBE option to ignore the number of available slots and size of the pe-list when placing the processes. # prrte-4.1.1~rc2/src/mca/ras/0000775000175000017500000000000015203542132015753 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/simulator/0000775000175000017500000000000015203542132017772 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/simulator/owner.txt0000664000175000017500000000024415203542132021665 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ras/simulator/ras_sim.h0000664000175000017500000000177615203542132021613 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_RAS_SIM_H #define PRTE_RAS_SIM_H #include "prte_config.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS struct prte_ras_sim_component_t { prte_ras_base_component_t super; char *num_nodes; char *slots; char *slots_max; char *topofiles; char *topologies; bool have_cpubind; bool have_membind; }; typedef struct prte_ras_sim_component_t prte_ras_sim_component_t; PRTE_EXPORT extern prte_ras_sim_component_t prte_mca_ras_simulator_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_sim_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/simulator/ras_sim_module.c0000664000175000017500000001377315203542132023153 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include #include #include "src/class/pmix_list.h" #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_show_help.h" #include "ras_sim.h" /* * Local functions */ static int allocate(prte_job_t *jdata, pmix_list_t *nodes); static int finalize(void); /* * Global variable */ prte_ras_base_module_t prte_ras_sim_module = { .init = NULL, .allocate = allocate, .deallocate = NULL, .modify = NULL, .finalize = finalize }; static int allocate(prte_job_t *jdata, pmix_list_t *nodes) { int i, n, val, dig, num_nodes, nslots; prte_node_t *node; prte_topology_t *t; hwloc_topology_t topo; hwloc_obj_t obj; char **node_cnt = NULL; char **slot_cnt = NULL; char **max_slot_cnt = NULL; char *tmp, *job_cpuset = NULL; char prefix[6]; bool use_hwthread_cpus = false; hwloc_cpuset_t available; node_cnt = PMIx_Argv_split(prte_mca_ras_simulator_component.num_nodes, ','); num_nodes = PMIx_Argv_count(node_cnt); if (NULL != prte_mca_ras_simulator_component.slots) { slot_cnt = PMIx_Argv_split(prte_mca_ras_simulator_component.slots, ','); /* if they didn't provide a slot count for each node, then * backfill the slot_cnt so every node has a cnt */ nslots = PMIx_Argv_count(slot_cnt); if (nslots < num_nodes) { // take the last one given and extend it to cover remaining nodes tmp = slot_cnt[nslots - 1]; for (n = nslots; n < num_nodes; n++) { PMIx_Argv_append_nosize(&slot_cnt, tmp); } } } if (NULL != prte_mca_ras_simulator_component.slots_max) { max_slot_cnt = PMIx_Argv_split(prte_mca_ras_simulator_component.slots_max, ','); /* if they didn't provide a max slot count for each node, then * backfill the slot_cnt so every node has a cnt */ nslots = PMIx_Argv_count(max_slot_cnt); if (nslots < num_nodes) { // take the last one given and extend it to cover remaining nodes tmp = max_slot_cnt[nslots - 1]; for (n = nslots; n < num_nodes; n++) { PMIx_Argv_append_nosize(&max_slot_cnt, tmp); } } } /* setup the prefix to the node names */ snprintf(prefix, 6, "nodeA"); /* see if this job has a "soft" cgroup assignment */ job_cpuset = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_CPUSET, (void **) &job_cpuset, PMIX_STRING)) { job_cpuset = NULL; } /* see if they want are using hwthreads as cpus */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { use_hwthread_cpus = true; } else { use_hwthread_cpus = false; } /* use our topology */ t = (prte_topology_t *) pmix_pointer_array_get_item(prte_node_topologies, 0); if (NULL == t) { return PRTE_ERR_NOT_FOUND; } topo = t->topo; if (NULL != job_cpuset) { available = prte_hwloc_base_generate_cpuset(topo, use_hwthread_cpus, &job_cpuset); } else { available = prte_hwloc_base_filter_cpus(topo); } /* process the request */ for (n = 0; NULL != node_cnt[n]; n++) { num_nodes = strtol(node_cnt[n], NULL, 10); /* get number of digits */ val = num_nodes; for (dig = 0; 0 != val; dig++) { val /= 10; } /* set the prefix for this group of nodes */ prefix[4] += n; for (i = 0; i < num_nodes; i++) { node = PMIX_NEW(prte_node_t); pmix_asprintf(&node->name, "%s%0*d", prefix, dig, i); node->state = PRTE_NODE_STATE_UP; node->slots_inuse = 0; if (NULL == slot_cnt || NULL == slot_cnt[n]) { obj = hwloc_get_root_obj(t->topo); node->slots = prte_hwloc_base_get_npus(t->topo, use_hwthread_cpus, available, obj); } else { node->slots = strtol(slot_cnt[n], NULL, 10); } if (NULL == max_slot_cnt || NULL == max_slot_cnt[n]) { obj = hwloc_get_root_obj(t->topo); node->slots_max = prte_hwloc_base_get_npus(t->topo, use_hwthread_cpus, available, obj); } else { node->slots_max = strtol(max_slot_cnt[n], NULL, 10); } PMIX_RETAIN(t); node->topology = t; pmix_output_verbose(1, prte_ras_base_framework.framework_output, "Created Node <%10s> [%3d : %3d]", node->name, node->slots, node->slots_max); node->available = hwloc_bitmap_dup(available); pmix_list_append(nodes, &node->super); } } hwloc_bitmap_free(available); /* record the number of allocated nodes */ prte_num_allocated_nodes = pmix_list_get_size(nodes); // ensure we do not attempt to launch this job prte_set_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); if (NULL != max_slot_cnt) { PMIx_Argv_free(max_slot_cnt); } if (NULL != slot_cnt) { PMIx_Argv_free(slot_cnt); } PMIx_Argv_free(node_cnt); if (NULL != job_cpuset) { free(job_cpuset); } return PRTE_SUCCESS; } /* * There's really nothing to do here */ static int finalize(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/simulator/ras_sim_component.c0000664000175000017500000001126215203542132023657 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "ras_sim.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int ras_sim_register(void); static int ras_sim_component_query(pmix_mca_base_module_t **module, int *priority); prte_ras_sim_component_t prte_mca_ras_simulator_component = { .super = { PRTE_RAS_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "simulator", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_query_component = ras_sim_component_query, .pmix_mca_register_component_params = ras_sim_register } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, simulator) static int ras_sim_register(void) { pmix_mca_base_component_t *component = &prte_mca_ras_simulator_component.super; prte_mca_ras_simulator_component.slots = NULL; (void) pmix_mca_base_component_var_register(component, "slots", "Comma-separated list of number of slots on each node to simulate", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_ras_simulator_component.slots); prte_mca_ras_simulator_component.slots_max = NULL; (void) pmix_mca_base_component_var_register(component, "max_slots", "Comma-separated list of number of max slots on each node to simulate", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_ras_simulator_component.slots_max); prte_mca_ras_simulator_component.num_nodes = NULL; (void) pmix_mca_base_component_var_register(component, "num_nodes", "Comma-separated list of number of nodes to simulate for each topology", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_ras_simulator_component.num_nodes); prte_mca_ras_simulator_component.have_cpubind = true; (void) pmix_mca_base_component_var_register(component, "have_cpubind", "Topology supports binding to cpus", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_mca_ras_simulator_component.have_cpubind); prte_mca_ras_simulator_component.have_membind = true; (void) pmix_mca_base_component_var_register(component, "have_membind", "Topology supports binding to memory", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_mca_ras_simulator_component.have_membind); return PRTE_SUCCESS; } static int ras_sim_component_query(pmix_mca_base_module_t **module, int *priority) { if (NULL != prte_mca_ras_simulator_component.num_nodes) { *module = (pmix_mca_base_module_t *) &prte_ras_sim_module; *priority = 1000; prte_ras_base.simulated = true; return PRTE_SUCCESS; } /* Sadly, no */ *module = NULL; *priority = 0; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/ras/simulator/Makefile.am0000664000175000017500000000225415203542132022031 0ustar alastairalastair# # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ras_sim.h \ ras_sim_component.c \ ras_sim_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_simulator_DSO lib = lib_sources = component = prte_mca_ras_simulator.la component_sources = $(sources) else lib = libprtemca_ras_simulator.la lib_sources = $(sources) component = component_sources = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component) prte_mca_ras_simulator_la_SOURCES = $(component_sources) prte_mca_ras_simulator_la_LDFLAGS = -module -avoid-version prte_mca_ras_simulator_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(lib) libprtemca_ras_simulator_la_SOURCES = $(lib_sources) libprtemca_ras_simulator_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ras/lsf/0000775000175000017500000000000015203542132016537 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/lsf/help-ras-lsf.txt0000664000175000017500000000221315203542132021573 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English help file for PRTE MCA error messages. # [nodelist-failed] While trying to determine what resources are available, LSF failed when queried for a list of available nodes. This may indicate a problem with LSF or your cluster. prrte-4.1.1~rc2/src/mca/ras/lsf/owner.txt0000664000175000017500000000024415203542132020432 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ras/lsf/ras_lsf.h0000664000175000017500000000270015203542132020340 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Allocation (LSF) */ #ifndef PRTE_RAS_LSF_H #define PRTE_RAS_LSF_H #include "prte_config.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS /** * RAS Component */ PRTE_EXPORT extern prte_ras_base_component_t prte_mca_ras_lsf_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_lsf_module; PRTE_EXPORT extern bool prte_ras_lsf_skip_affinity_file; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/lsf/ras_lsf_component.c0000664000175000017500000000676115203542132022430 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #if PRTE_TESTBUILD_LAUNCHERS #include "testbuild_lsf.h" #else #include #endif #include "src/mca/base/pmix_base.h" #include "ras_lsf.h" /* * Local functions */ static int prte_ras_lsf_open(void); static int prte_ras_lsf_close(void); static int prte_mca_ras_lsf_component_query(pmix_mca_base_module_t **module, int *priority); static int prte_ras_lsf_register(void); bool prte_ras_lsf_skip_affinity_file = false; prte_ras_base_component_t prte_mca_ras_lsf_component = { /* Indicate that we are a ras v2.0.0 component (which also implies a specific MCA version) */ PRTE_RAS_BASE_VERSION_2_0_0, .pmix_mca_component_name = "lsf", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PRTE_RELEASE_VERSION), .pmix_mca_open_component = prte_ras_lsf_open, .pmix_mca_close_component = prte_ras_lsf_close, .pmix_mca_query_component = prte_mca_ras_lsf_component_query, .pmix_mca_register_component_params = prte_ras_lsf_register, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, lsf) /** * component open/close/init function */ static int prte_ras_lsf_open(void) { return PRTE_SUCCESS; } static int prte_mca_ras_lsf_component_query(pmix_mca_base_module_t **module, int *priority) { /* check if lsf is running here */ if (NULL == getenv("LSB_JOBID") || lsb_init("PRTE launcher") < 0) { /* nope, not here */ *module = NULL; return PRTE_ERROR; } *priority = 75; *module = (pmix_mca_base_module_t *) &prte_ras_lsf_module; return PRTE_SUCCESS; } /** * Close all subsystems. */ static int prte_ras_lsf_close(void) { return PRTE_SUCCESS; } static int prte_ras_lsf_register(void) { prte_ras_lsf_skip_affinity_file = false; (void) pmix_mca_base_component_var_register(&prte_mca_ras_lsf_component, "skip_affinity_file", "Skip processing the LSB_AFFINITY_HOSTFILE.", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_ras_lsf_skip_affinity_file); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/lsf/configure.m40000664000175000017500000000315315203542132020764 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ras_lsf_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ras_lsf_CONFIG],[ AC_CONFIG_FILES([src/mca/ras/lsf/Makefile]) PRTE_CHECK_LSF([ras_lsf], [ras_lsf_good=1], [ras_lsf_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$ras_lsf_good" = "1" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([ras_lsf_CPPFLAGS]) AC_SUBST([ras_lsf_LDFLAGS]) AC_SUBST([ras_lsf_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/ras/lsf/ras_lsf_module.c0000664000175000017500000000655615203542132021715 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2022 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #include #include #include #define SR1_PJOBS #if PRTE_TESTBUILD_LAUNCHERS #include "testbuild_lsf.h" #else #include #endif #include "src/hwloc/hwloc-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_net.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_show_help.h" #include "ras_lsf.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int allocate(prte_job_t *jdata, pmix_list_t *nodes); static int finalize(void); /* * Global variable */ prte_ras_base_module_t prte_ras_lsf_module = { .init = NULL, .allocate = allocate, .finalize = finalize }; static int allocate(prte_job_t *jdata, pmix_list_t *nodes) { char **nodelist = NULL; prte_node_t *node; int i, num_nodes; PRTE_HIDE_UNUSED_PARAMS(jdata); /* get the list of allocated nodes */ if ((num_nodes = lsb_getalloc(&nodelist)) < 0) { pmix_show_help("help-ras-lsf.txt", "nodelist-failed", true); return PRTE_ERR_NOT_AVAILABLE; } node = NULL; /* step through the list */ for (i = 0; i < num_nodes; i++) { /* is this a repeat of the current node? */ if (NULL != node && 0 == strcmp(nodelist[i], node->name)) { /* it is a repeat - just bump the slot count */ ++node->slots; pmix_output_verbose(10, prte_ras_base_framework.framework_output, "ras/lsf: +++ Node (%s) [slots=%d]", node->name, node->slots); continue; } /* not a repeat - create a node entry for it */ node = PMIX_NEW(prte_node_t); node->name = strdup(nodelist[i]); node->slots_inuse = 0; node->slots_max = 0; node->slots = 1; node->state = PRTE_NODE_STATE_UP; pmix_list_append(nodes, &node->super); pmix_output_verbose(10, prte_ras_base_framework.framework_output, "ras/lsf: New Node (%s) [slots=%d]", node->name, node->slots); } /* release the nodelist from lsf */ PMIx_Argv_free(nodelist); return PRTE_SUCCESS; } static int finalize(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/lsf/testbuild_lsf.h0000664000175000017500000000247615203542132021564 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_RAS_LSF_TESTBUILD_H #define PRTE_RAS_LSF_TESTBUILD_H #include "prte_config.h" #include "src/mca/mca.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS int lsb_init(char *str); int lsb_getalloc(char ***hostlist); END_C_DECLS #endif /* PRTE_RAS_LSF_TESTBUILD_H */ prrte-4.1.1~rc2/src/mca/ras/lsf/Makefile.am0000664000175000017500000000362715203542132020603 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = help-ras-lsf.txt # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_lsf_DSO component_noinst = component_install = prte_mca_ras_lsf.la else component_noinst = libprtemca_ras_lsf.la component_install = endif AM_CPPFLAGS= $(ras_lsf_CPPFLAGS) proxy_SOURCES = \ ras_lsf.h \ ras_lsf_module.c \ ras_lsf_component.c mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ras_lsf_la_SOURCES = $(proxy_SOURCES) prte_mca_ras_lsf_la_LIBADD = $(ras_lsf_LIBS) \ $(top_builddir)/src/libprrte.la prte_mca_ras_lsf_la_LDFLAGS = -module -avoid-version $(ras_lsf_LDFLAGS) noinst_LTLIBRARIES = $(component_noinst) libprtemca_ras_lsf_la_SOURCES = $(proxy_SOURCES) libprtemca_ras_lsf_la_LIBADD = $(ras_lsf_LIBS) libprtemca_ras_lsf_la_LDFLAGS = -module -avoid-version $(ras_lsf_LDFLAGS) prrte-4.1.1~rc2/src/mca/ras/slurm/0000775000175000017500000000000015203542132017115 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/slurm/ras_slurm.h0000664000175000017500000000321315203542132021274 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Allocation (SLURM) */ #ifndef PRTE_RAS_SLURM_H #define PRTE_RAS_SLURM_H #include "prte_config.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS typedef struct { prte_ras_base_component_t super; int max_length; bool use_all; } prte_mca_ras_slurm_component_t; PRTE_EXPORT extern prte_mca_ras_slurm_component_t prte_mca_ras_slurm_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_slurm_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/slurm/owner.txt0000664000175000017500000000024415203542132021010 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ras/slurm/ras_slurm_module.c0000664000175000017500000004531215203542132022642 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include #include #include #include #include #ifdef HAVE_NETINET_IN_H # include #endif #ifdef HAVE_ARPA_INET_H # include #endif #include #include #include #include "src/include/prte_socket_errno.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "ras_slurm.h" #include "src/mca/ras/base/base.h" #define PRTE_SLURM_DYN_MAX_SIZE 256 /* * API functions */ static int init(void); static int prte_ras_slurm_allocate(prte_job_t *jdata, pmix_list_t *nodes); static void deallocate(prte_job_t *jdata, prte_app_context_t *app); static void modify(prte_pmix_server_req_t *req); static int prte_ras_slurm_finalize(void); /* * RAS slurm module */ prte_ras_base_module_t prte_ras_slurm_module = { .init = init, .allocate = prte_ras_slurm_allocate, .deallocate = deallocate, .modify = modify, .finalize = prte_ras_slurm_finalize }; /* Local functions */ static int prte_ras_slurm_discover(char *regexp, char *tasks_per_node, pmix_list_t *nodelist); static int prte_ras_slurm_parse_ranges(char *base, char *ranges, char ***nodelist); static int prte_ras_slurm_parse_range(char *base, char *range, char ***nodelist); static bool check_taint(char *name, char *evar) { int n; for (n=0; n < prte_mca_ras_slurm_component.max_length; n++) { if ('\0' == evar[n]) { return false; } } pmix_show_help("help-ras-slurm.txt", "tainted-envar", true, name, prte_mca_ras_slurm_component.max_length); return true; } /* init the module */ static int init(void) { return PRTE_SUCCESS; } /** * Discover available (pre-allocated) nodes. Allocate the * requested number of nodes/process slots to the job. * */ static int prte_ras_slurm_allocate(prte_job_t *jdata, pmix_list_t *nodes) { int ret, cpus_per_task; char *regexp; char *tasks_per_node, *node_tasks; char *tmp; char *slurm_jobid; PRTE_HIDE_UNUSED_PARAMS(jdata); if (NULL == (slurm_jobid = getenv("SLURM_JOBID"))) { return PRTE_ERR_TAKE_NEXT_OPTION; } regexp = getenv("SLURM_NODELIST"); if (NULL == regexp) { pmix_show_help("help-ras-slurm.txt", "slurm-env-var-not-found", 1, "SLURM_NODELIST"); return PRTE_ERR_NOT_FOUND; } // check for length violation - untaint the envar value if (check_taint("SLURM_NODELIST", regexp)) { return PRTE_ERR_BAD_PARAM; } if (prte_mca_ras_slurm_component.use_all) { /* this is an oddball case required for debug situations where * a tool is started that will then call mpirun. In this case, * Slurm will assign only 1 tasks/per node to the tool, but * we want mpirun to use the entire allocation. They don't give * us a specific variable for this purpose, so we have to fudge * a bit - but this is a special edge case, and we'll live with it */ tasks_per_node = getenv("SLURM_JOB_CPUS_PER_NODE"); if (NULL == tasks_per_node) { /* couldn't find any version - abort */ pmix_show_help("help-ras-slurm.txt", "slurm-env-var-not-found", 1, "SLURM_JOB_CPUS_PER_NODE"); return PRTE_ERR_NOT_FOUND; } if (check_taint("SLURM_JOB_CPUS_PER_NODE", tasks_per_node)) { return PRTE_ERR_BAD_PARAM; } node_tasks = strdup(tasks_per_node); if (NULL == node_tasks) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } cpus_per_task = 1; } else { /* get the number of process slots we were assigned on each node */ tasks_per_node = getenv("SLURM_TASKS_PER_NODE"); if (NULL == tasks_per_node) { /* couldn't find any version - abort */ pmix_show_help("help-ras-slurm.txt", "slurm-env-var-not-found", 1, "SLURM_TASKS_PER_NODE"); return PRTE_ERR_NOT_FOUND; } if (check_taint("SLURM_TASKS_PER_NODE", tasks_per_node)) { return PRTE_ERR_BAD_PARAM; } node_tasks = strdup(tasks_per_node); if (NULL == node_tasks) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } /* get the number of CPUs per task that the user provided to slurm */ tmp = getenv("SLURM_CPUS_PER_TASK"); if (NULL != tmp) { if (check_taint("SLURM_CPUS_PER_TASK", tmp)) { free(node_tasks); return PRTE_ERR_BAD_PARAM; } cpus_per_task = atoi(tmp); if (0 >= cpus_per_task) { pmix_output(0, "ras:slurm:allocate: Got bad value from SLURM_CPUS_PER_TASK. " "Variable was: %s\n", tmp); PRTE_ERROR_LOG(PRTE_ERROR); free(node_tasks); return PRTE_ERROR; } } else { cpus_per_task = 1; } } ret = prte_ras_slurm_discover(regexp, node_tasks, nodes); free(node_tasks); if (PRTE_SUCCESS != ret) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate: discover failed!", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); return ret; } /* record the number of allocated nodes */ prte_num_allocated_nodes = pmix_list_get_size(nodes); /* All done */ PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate: success", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); return PRTE_SUCCESS; } static void deallocate(prte_job_t *jdata, prte_app_context_t *app) { PRTE_HIDE_UNUSED_PARAMS(jdata, app); return; } static void modify(prte_pmix_server_req_t *req) { req->status = PMIX_ERR_NOT_SUPPORTED; return; } static int prte_ras_slurm_finalize(void) { return PRTE_SUCCESS; } /** * Discover the available resources. * * In order to fully support slurm, we need to be able to handle * node regexp/task_per_node strings such as: * foo,bar 5,3 * foo 5 * foo[2-10,12,99-105],bar,foobar[3-11] 2(x10),5,100(x16) * * @param *regexp A node regular expression from SLURM (i.e. SLURM_NODELIST) * @param *tasks_per_node A tasks per node expression from SLURM * (i.e. SLURM_TASKS_PER_NODE) * @param *nodelist A list which has already been constucted to return * the found nodes in */ static int prte_ras_slurm_discover(char *regexp, char *tasks_per_node, pmix_list_t *nodelist) { int i, j, len, ret, count, reps, num_nodes; char *base, **names = NULL; char *begptr, *endptr, *orig; int *slots; bool found_range = false; bool more_to_come = false; orig = base = strdup(regexp); if (NULL == base) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate:discover: checking nodelist: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), regexp)); do { /* Find the base */ len = strlen(base); for (i = 0; i <= len; ++i) { if (base[i] == '[') { /* we found a range. this gets dealt with below */ base[i] = '\0'; found_range = true; break; } if (base[i] == ',') { /* we found a singleton node, and there are more to come */ base[i] = '\0'; found_range = false; more_to_come = true; break; } if (base[i] == '\0') { /* we found a singleton node */ found_range = false; more_to_come = false; break; } } if (i == 0) { /* we found a special character at the beginning of the string */ pmix_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1, regexp, tasks_per_node, "SLURM_NODELIST"); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); free(orig); return PRTE_ERR_BAD_PARAM; } if (found_range) { /* If we found a range, now find the end of the range */ for (j = i; j < len; ++j) { if (base[j] == ']') { base[j] = '\0'; break; } } if (j >= len) { /* we didn't find the end of the range */ pmix_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1, regexp, tasks_per_node, "SLURM_NODELIST"); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); free(orig); return PRTE_ERR_BAD_PARAM; } ret = prte_ras_slurm_parse_ranges(base, base + i + 1, &names); if (PRTE_SUCCESS != ret) { pmix_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1, regexp, tasks_per_node, "SLURM_NODELIST"); PRTE_ERROR_LOG(ret); free(orig); return ret; } if (base[j + 1] == ',') { more_to_come = true; base = &base[j + 2]; } else { more_to_come = false; } } else { /* If we didn't find a range, just add the node */ PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate:discover: found node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), base)); if (PRTE_SUCCESS != (ret = PMIx_Argv_append_nosize(&names, base))) { PRTE_ERROR_LOG(ret); free(orig); return ret; } /* set base equal to the (possible) next base to look at */ base = &base[i + 1]; } } while (more_to_come); free(orig); num_nodes = PMIx_Argv_count(names); /* Find the number of slots per node */ slots = malloc(sizeof(int) * num_nodes); if (NULL == slots) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } memset(slots, 0, sizeof(int) * num_nodes); orig = begptr = strdup(tasks_per_node); if (NULL == begptr) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); free(slots); return PRTE_ERR_OUT_OF_RESOURCE; } j = 0; while (begptr) { count = strtol(begptr, &endptr, 10); if ((endptr[0] == '(') && (endptr[1] == 'x')) { reps = strtol((endptr + 2), &endptr, 10); if (endptr[0] == ')') { endptr++; } } else { reps = 1; } /** * TBP: it seems like it would be an error to have more slot * descriptions than nodes. Turns out that this valid, and SLURM will * return such a thing. For instance, if I did: * srun -A -N 30 -w odin001 * I would get SLURM_NODELIST=odin001 SLURM_TASKS_PER_NODE=4(x30) * That is, I am allocated 30 nodes, but since I only requested * one specific node, that's what is in the nodelist. * I'm not sure this is what users would expect, but I think it is * more of a SLURM issue than a prte issue, since SLURM is OK with it, * I'm ok with it */ for (i = 0; i < reps && j < num_nodes; i++) { slots[j++] = count; } if (*endptr == ',') { begptr = endptr + 1; } else if (*endptr == '\0' || j >= num_nodes) { break; } else { pmix_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1, regexp, tasks_per_node, "SLURM_TASKS_PER_NODE"); PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); free(slots); free(orig); return PRTE_ERR_BAD_PARAM; } } free(orig); /* Convert the argv of node names to a list of node_t's */ for (i = 0; NULL != names && NULL != names[i]; ++i) { prte_node_t *node; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate:discover: adding node %s (%d slot%s)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), names[i], slots[i], (1 == slots[i]) ? "" : "s")); node = PMIX_NEW(prte_node_t); if (NULL == node) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); free(slots); return PRTE_ERR_OUT_OF_RESOURCE; } node->name = strdup(names[i]); node->state = PRTE_NODE_STATE_UP; node->slots_inuse = 0; node->slots_max = 0; node->slots = slots[i]; pmix_list_append(nodelist, &node->super); } free(slots); PMIx_Argv_free(names); /* All done */ return ret; } /* * Parse one or more ranges in a set * * @param base The base text of the node name * @param *ranges A pointer to a range. This can contain multiple ranges * (i.e. "1-3,10" or "5" or "9,0100-0130,250") * @param ***names An argv array to add the newly discovered nodes to */ static int prte_ras_slurm_parse_ranges(char *base, char *ranges, char ***names) { int i, len, ret; char *start, *orig; /* Look for commas, the separator between ranges */ len = strlen(ranges); for (orig = start = ranges, i = 0; i < len; ++i) { if (',' == ranges[i]) { ranges[i] = '\0'; ret = prte_ras_slurm_parse_range(base, start, names); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); return ret; } start = ranges + i + 1; } } /* Pick up the last range, if it exists */ if (start < orig + len) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:slurm:allocate:discover: parse range %s (2)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), start)); ret = prte_ras_slurm_parse_range(base, start, names); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); return ret; } } /* All done */ return PRTE_SUCCESS; } /* * Parse a single range in a set and add the full names of the nodes * found to the names argv * * @param base The base text of the node name * @param *ranges A pointer to a single range. (i.e. "1-3" or "5") * @param ***names An argv array to add the newly discovered nodes to */ static int prte_ras_slurm_parse_range(char *base, char *range, char ***names) { char *str, temp1[BUFSIZ]; size_t i, j, start, end; size_t base_len, len, num_len; size_t num_str_len; bool found; int ret; len = strlen(range); base_len = strlen(base); /* Silence compiler warnings; start and end are always assigned properly, below */ start = end = 0; /* Look for the beginning of the first number */ for (found = false, i = 0; i < len; ++i) { if (isdigit((int) range[i])) { if (!found) { start = atoi(range + i); found = true; break; } } } if (!found) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } /* Look for the end of the first number */ for (found = false, num_str_len = 0; i < len; ++i, ++num_str_len) { if (!isdigit((int) range[i])) { break; } } /* Was there no range, just a single number? */ if (i >= len) { end = start; found = true; } /* Nope, there was a range. Look for the beginning of the second number */ else { for (; i < len; ++i) { if (isdigit((int) range[i])) { end = atoi(range + i); found = true; break; } } } if (!found) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } /* Make strings for all values in the range */ len = base_len + num_str_len + 32; str = malloc(len); if (NULL == str) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return PRTE_ERR_OUT_OF_RESOURCE; } strcpy(str, base); for (i = start; i <= end; ++i) { str[base_len] = '\0'; snprintf(temp1, BUFSIZ - 1, "%lu", (long) i); /* Do we need zero pading? */ if ((num_len = strlen(temp1)) < num_str_len) { for (j = base_len; j < base_len + (num_str_len - num_len); ++j) { str[j] = '0'; } str[j] = '\0'; } strcat(str, temp1); ret = PMIx_Argv_append_nosize(names, str); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); free(str); return ret; } } free(str); /* All done */ return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/slurm/configure.m40000664000175000017500000000276415203542132021351 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ras_slurm_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ras_slurm_CONFIG],[ AC_CONFIG_FILES([src/mca/ras/slurm/Makefile]) PRTE_CHECK_SLURM([ras_slurm], [ras_slurm_good=1], [ras_slurm_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$ras_slurm_good" = "1" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) ])dnl prrte-4.1.1~rc2/src/mca/ras/slurm/help-ras-slurm.txt0000664000175000017500000000405015203542132022530 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2024-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English help file for PRTE MCA error messages. # [slurm-env-var-not-found] While trying to determine what resources are available, the SLURM resource allocator expects to find the following environment variables: SLURM_NODELIST SLURM_TASKS_PER_NODE However, it was unable to find the following environment variable: %s This is a fatal error. # [slurm-env-var-bad-value] While trying to determine what resources are available, the SLURM resource allocator uses the following environment variables: SLURM_NODELIST value: %s SLURM_TASKS_PER_NODE value: %s However, an error was encountered when trying to parse the following variable: %s This is a fatal error. # [tainted-envar] While trying to determine what resources are available, the SLURM resource allocator expects to find the following environment variable: Envar: %s The variable was found, but a length check has failed, indicating that the envar is more than %d characters long. This indicates that the envar may be tainted. We will therefore not print it out. However, if you check and feel it is valid, then please adjust the length limit and try again. prrte-4.1.1~rc2/src/mca/ras/slurm/ras_slurm_component.c0000664000175000017500000001075015203542132023355 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/include/prte_socket_errno.h" #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/util/pmix_net.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "ras_slurm.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int ras_slurm_register(void); static int ras_slurm_open(void); static int ras_slurm_close(void); static int prte_mca_ras_slurm_component_query(pmix_mca_base_module_t **module, int *priority); prte_mca_ras_slurm_component_t prte_mca_ras_slurm_component = { .super = { PRTE_RAS_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "slurm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = ras_slurm_open, .pmix_mca_close_component = ras_slurm_close, .pmix_mca_query_component = prte_mca_ras_slurm_component_query, .pmix_mca_register_component_params = ras_slurm_register } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, slurm) static int ras_slurm_register(void) { pmix_mca_base_component_t *component = &prte_mca_ras_slurm_component.super; prte_mca_ras_slurm_component.max_length = 32000; (void) pmix_mca_base_component_var_register(component, "max_envar_length", "Maximum length of the SLURM_NODELIST envar we should allow", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_mca_ras_slurm_component.max_length); prte_mca_ras_slurm_component.use_all = false; (void) pmix_mca_base_component_var_register(component, "use_entire_allocation", "Use entire allocation (not just job step nodes) for this application", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_mca_ras_slurm_component.use_all); return PRTE_SUCCESS; } static int ras_slurm_open(void) { return PRTE_SUCCESS; } static int ras_slurm_close(void) { return PRTE_SUCCESS; } static int prte_mca_ras_slurm_component_query(pmix_mca_base_module_t **module, int *priority) { /* if I built, then slurm support is available. If * I am not in a Slurm allocation, and dynamic alloc * is not enabled, then disqualify myself */ if (NULL == getenv("SLURM_JOBID")) { /* disqualify ourselves */ *priority = 0; *module = NULL; return PRTE_ERROR; } PMIX_OUTPUT_VERBOSE((2, prte_ras_base_framework.framework_output, "%s ras:slurm: available for selection", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* since only one RM can exist on a cluster, just set * my priority to something - the other components won't * be responding anyway */ *priority = 50; *module = (pmix_mca_base_module_t *) &prte_ras_slurm_module; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/slurm/Makefile.am0000664000175000017500000000366415203542132021162 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # The Slurm plugin does not call any Slurm functions - it therefore # does NOT need to link against any Slurm libraries EXTRA_DIST = \ help-ras-slurm.txt sources = \ ras_slurm.h \ ras_slurm_component.c \ ras_slurm_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_slurm_DSO lib = lib_sources = component = prte_mca_ras_slurm.la component_sources = $(sources) else lib = libprtemca_ras_slurm.la lib_sources = $(sources) component = component_sources = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component) prte_mca_ras_slurm_la_SOURCES = $(component_sources) prte_mca_ras_slurm_la_LDFLAGS = -module -avoid-version prte_mca_ras_slurm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(lib) libprtemca_ras_slurm_la_SOURCES = $(lib_sources) libprtemca_ras_slurm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ras/pbs/0000775000175000017500000000000015203542132016537 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/pbs/owner.txt0000664000175000017500000000024415203542132020432 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ras/pbs/ras_pbs_component.c0000664000175000017500000001020715203542132022416 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2007 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/util/pmix_basename.h" #include "ras_pbs.h" #include "src/mca/ras/base/base.h" /* * Local variables */ static int param_priority; /* * Local functions */ static int ras_pbs_register(void); static int ras_pbs_open(void); static int prte_mca_ras_pbs_component_query(pmix_mca_base_module_t **module, int *priority); prte_mca_ras_pbs_component_t prte_mca_ras_pbs_component = { .super = { PRTE_RAS_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "pbs", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = ras_pbs_open, .pmix_mca_query_component = prte_mca_ras_pbs_component_query, .pmix_mca_register_component_params = ras_pbs_register, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, pbs) static int ras_pbs_register(void) { pmix_mca_base_component_t *c = &prte_mca_ras_pbs_component.super; param_priority = 100; (void) pmix_mca_base_component_var_register(c, "priority", "Priority of the pbs ras component", PMIX_MCA_BASE_VAR_TYPE_INT, ¶m_priority); /* for big SMP machines (e.g., those from SGI), listing the nodes * once/slot in the nodefile is extreme. In those cases, they may * choose to list each node once, but then provide an envar that * tells us how many cpus/node were allocated. Allow the user to * inform us that we are in such an environment */ prte_mca_ras_pbs_component.smp_mode = false; (void) pmix_mca_base_component_var_register(c, "smp", "The PBS system is configured in SMP mode " "with the number of cpus/node given in the environment", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_mca_ras_pbs_component.smp_mode); return PRTE_SUCCESS; } static int ras_pbs_open(void) { return PRTE_SUCCESS; } static int prte_mca_ras_pbs_component_query(pmix_mca_base_module_t **module, int *priority) { /* Are we running under a PBS job? */ if (NULL != getenv("PBS_ENVIRONMENT") && NULL != getenv("PBS_JOBID")) { *priority = param_priority; *module = (pmix_mca_base_module_t *) &prte_ras_pbs_module; return PRTE_SUCCESS; } else if (NULL != getenv("COBALT_JOBID")) { /* we are running under Argonne's Cobalt variant */ *priority = param_priority; *module = (pmix_mca_base_module_t *) &prte_ras_pbs_module; return PRTE_SUCCESS; } /* Sadly, no */ *module = NULL; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/ras/pbs/configure.m40000664000175000017500000000336215203542132020766 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2026 Nanook Consulting All rights reserved. # Copyright (c) 2022 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2022 Amazon.com, Inc. or its affiliates. # All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ras_pbs_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ras_pbs_CONFIG],[ AC_CONFIG_FILES([src/mca/ras/pbs/Makefile]) PRTE_CHECK_PBS([ras_pbs], [ras_pbs_good=1], [ras_pbs_good=0]) AS_IF([test "$ras_pbs_good" = "1" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) # there are no libraries to add to the Makefile as we don't # link against any PBS libs ])dnl prrte-4.1.1~rc2/src/mca/ras/pbs/ras_pbs.h0000664000175000017500000000307715203542132020350 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Resource Allocation (PBS) */ #ifndef PRTE_RAS_PBS_H #define PRTE_RAS_PBS_H #include "prte_config.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS struct prte_mca_ras_pbs_component_t { prte_ras_base_component_t super; bool smp_mode; }; typedef struct prte_mca_ras_pbs_component_t prte_mca_ras_pbs_component_t; PRTE_EXPORT extern prte_mca_ras_pbs_component_t prte_mca_ras_pbs_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_pbs_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/pbs/Makefile.am0000664000175000017500000000371215203542132020576 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # The PBS plugin does not call any Torque/PBS functions - it therefore # does NOT need to link against any Torque/PBS libraries # Use the top-level Makefile.options EXTRA_DIST = \ help-ras-pbs.txt sources = \ ras_pbs.h \ ras_pbs_component.c \ ras_pbs_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_pbs_DSO lib = lib_sources = component = prte_mca_ras_pbs.la component_sources = $(sources) else lib = libprtemca_ras_pbs.la lib_sources = $(sources) component = component_sources = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component) prte_mca_ras_pbs_la_SOURCES = $(component_sources) prte_mca_ras_pbs_la_LDFLAGS = -module -avoid-version prte_mca_ras_pbs_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(lib) libprtemca_ras_pbs_la_SOURCES = $(lib_sources) libprtemca_ras_pbs_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ras/pbs/help-ras-pbs.txt0000664000175000017500000000324015203542132021574 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # Copyright (c) 2014-2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # [no-nodefile] The PBS_NODEFILE environmental variable was not found - no nodes can be allocated [no-nodes-found] No nodes were found in the PBS_NODEFILE: %s This is probably a setup issue, could be due to a corrupted file, or - most probably - is caused by having failed to obtain an allocation. # [smp-error] The SMP mode flag was given, but the required PBS_PPN environmental variable was not found. Please check that your system actually supports the indicated mode and try again. # [smp-multi] The SMP mode flag was given, but a node was listed in the PBS_Nodefile more than once. This usually indicates that the system is not actually operating in SMP mode. Please check your system configuration and try again. prrte-4.1.1~rc2/src/mca/ras/pbs/ras_pbs_module.c0000664000175000017500000001777215203542132021717 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include #include #include "src/util/pmix_net.h" #include "src/util/pmix_os_path.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "ras_pbs.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int allocate(prte_job_t *jdata, pmix_list_t *nodes); static int finalize(void); static int discover(pmix_list_t *nodelist, char *pbs_jobid); static char *pbs_getline(FILE *fp); #define PBS_FILE_MAX_LINE_LENGTH 512 static char *filename; /* * Global variable */ prte_ras_base_module_t prte_ras_pbs_module = { .allocate = allocate, .finalize = finalize }; /** * Discover available (pre-allocated) nodes and report * them back to the caller. * */ static int allocate(prte_job_t *jdata, pmix_list_t *nodes) { int ret; char *pbs_jobid; PRTE_HIDE_UNUSED_PARAMS(jdata); /* get our PBS jobid from the environment */ if (NULL == (pbs_jobid = getenv("PBS_JOBID"))) { /* see if we are in Argonne's Cobalt variant */ if (NULL == (pbs_jobid = getenv("COBALT_JOBID"))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } } /* save that value in the global job ident string for * later use in any error reporting */ prte_job_ident = strdup(pbs_jobid); if (PRTE_SUCCESS != (ret = discover(nodes, pbs_jobid))) { PRTE_ERROR_LOG(ret); return ret; } /* in the PBS world, if we didn't find anything, then this * is an unrecoverable error - report it */ if (pmix_list_is_empty(nodes)) { pmix_show_help("help-ras-pbs.txt", "no-nodes-found", true, filename); return PRTE_ERR_NOT_FOUND; } /* record the number of allocated nodes */ prte_num_allocated_nodes = pmix_list_get_size(nodes); /* All done */ return PRTE_SUCCESS; } /* * There's really nothing to do here */ static int finalize(void) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:pbs:finalize: success (nothing to do)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); return PRTE_SUCCESS; } /** * Discover the available resources. Obtain directly from PBS (and * therefore have no need to validate) -- ignore hostfile or any other * user-specified parameters. * * - validate any nodes specified via hostfile/commandline * - check for additional nodes that have already been allocated */ static int discover(pmix_list_t *nodelist, char *pbs_jobid) { int32_t nodeid; prte_node_t *node; FILE *fp; char *hostname, *cppn; int ppn; bool found; PRTE_HIDE_UNUSED_PARAMS(pbs_jobid); /* Ignore anything that the user already specified -- we're getting nodes only from PBS. */ /* PBS "nodes" may actually correspond to PBS "VCPUs", which means there may be multiple "PBS nodes" that correspond to the same physical node. This doesn't really affect what we're doing here (we actually ignore the fact that they're duplicates -- slightly inefficient, but no big deal); just mentioned for completeness... */ /* if we are in SMP mode, then read the environment to get the * number of cpus for each node read in the file */ if (prte_mca_ras_pbs_component.smp_mode) { if (NULL == (cppn = getenv("PBS_PPN"))) { pmix_show_help("help-ras-pbs.txt", "smp-error", true); return PRTE_ERR_NOT_FOUND; } ppn = strtol(cppn, NULL, 10); } else { ppn = 1; } /* setup the full path to the PBS file */ filename = getenv("PBS_NODEFILE"); if (NULL == filename) { /* try the Cobalt variant */ filename = getenv("COBALT_NODEFILE"); if (NULL == filename) { pmix_show_help("help-ras-pbs.txt", "no-nodefile", true); return PRTE_ERR_NOT_FOUND; } } fp = fopen(filename, "r"); if (NULL == fp) { PRTE_ERROR_LOG(PRTE_ERR_FILE_OPEN_FAILURE); return PRTE_ERR_FILE_OPEN_FAILURE; } /* Iterate through all the nodes and make an entry for each. PBS node ID's will never be duplicated, but they may end up resolving to the same hostname (i.e., vcpu's on a single host). */ nodeid = 0; while (NULL != (hostname = pbs_getline(fp))) { PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:pbs:allocate:discover: got hostname %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hostname)); /* Remember that PBS may list the same node more than once. So we have to check for duplicates. */ found = false; PMIX_LIST_FOREACH(node, nodelist, prte_node_t) { if (0 == strcmp(node->name, hostname)) { if (prte_mca_ras_pbs_component.smp_mode) { /* this cannot happen in smp mode */ pmix_show_help("help-ras-pbs.txt", "smp-multi", true); fclose(fp); free(hostname); return PRTE_ERR_BAD_PARAM; } ++node->slots; PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:pbs:allocate:discover: found -- bumped slots to %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->slots)); found = true; break; } } /* Did we find it? */ if (!found) { /* Nope -- didn't find it, so add a new item to the list */ PMIX_OUTPUT_VERBOSE((1, prte_ras_base_framework.framework_output, "%s ras:pbs:allocate:discover: not found -- added to list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); node = PMIX_NEW(prte_node_t); node->name = hostname; prte_set_attribute(&node->attributes, PRTE_NODE_LAUNCH_ID, PRTE_ATTR_LOCAL, &nodeid, PMIX_INT32); node->slots_inuse = 0; node->slots_max = 0; node->slots = ppn; node->state = PRTE_NODE_STATE_UP; pmix_list_append(nodelist, &node->super); } else { /* Yes, so we need to free the hostname that came back */ free(hostname); } /* up the nodeid */ nodeid++; } fclose(fp); return PRTE_SUCCESS; } static char *pbs_getline(FILE *fp) { char *ret, *buff; char input[PBS_FILE_MAX_LINE_LENGTH]; ret = fgets(input, PBS_FILE_MAX_LINE_LENGTH, fp); if (NULL != ret) { input[strlen(input) - 1] = '\0'; /* remove newline */ buff = strdup(input); return buff; } return NULL; } prrte-4.1.1~rc2/src/mca/ras/base/0000775000175000017500000000000015203542132016665 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/base/ras_base_frame.c0000664000175000017500000000723515203542132021771 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/event/event-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/runtime/prte_globals.h" #include "src/mca/ras/base/base.h" /* NOTE: the RAS does not require a proxy as only the * HNP can open the framework in prte_init - non-HNP * procs are not allowed to allocate resources */ /* * The following file was created by configure. It contains extern * statements and the definition of an array of pointers to each * component's public pmix_mca_base_component_t struct. */ #include "src/mca/ras/base/static-components.h" /* * Global variables */ prte_ras_base_t prte_ras_base = { .first_pass_completed = false, .allocation_read = false, .active_module = NULL, .total_slots_alloc = 0, .multiplier = 0, .launch_orted_on_hn = false, .simulated = false }; static int ras_register(pmix_mca_base_register_flag_t flags) { PRTE_HIDE_UNUSED_PARAMS(flags); prte_ras_base.multiplier = 1; pmix_mca_base_var_register("prte", "ras", "base", "multiplier", "Simulate a larger cluster by launching N daemons/node", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_ras_base.multiplier); prte_ras_base.launch_orted_on_hn = false; pmix_mca_base_var_register("prte", "ras", "base", "launch_orted_on_hn", "Launch an prte daemon on the head node", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_ras_base.launch_orted_on_hn); return PRTE_SUCCESS; } static int prte_ras_base_close(void) { /* Close selected component */ if (NULL != prte_ras_base.active_module) { prte_ras_base.active_module->finalize(); } return pmix_mca_base_framework_components_close(&prte_ras_base_framework, NULL); } /** * * Function for finding and opening either all MCA components, or the one * * that was specifically requested via a MCA parameter. * */ static int prte_ras_base_open(pmix_mca_base_open_flag_t flags) { /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_ras_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, ras, "PRTE Resource Allocation Subsystem", ras_register, prte_ras_base_open, prte_ras_base_close, prte_ras_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); prrte-4.1.1~rc2/src/mca/ras/base/owner.txt0000664000175000017500000000024415203542132020560 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ras/base/help-ras-base.txt0000664000175000017500000000430315203542132022051 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2024 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2023-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for the RAS base. # [ras-base:no-allocation] We were unable to find an allocation for this job as required by setting the "allocation required" flag. Please ensure you have the necessary allocation before executing again. If you wish to execute without a provided allocation (e.g., by providing a user-specified hostfile), please ensure that the "allocation required" flag is not set. This flag can be set in several forms, so please check that none of the following exist: MCA param file: prte_allocation_required = 1 Environment: PRTE_MCA_prte_allocation_required=1 Cmd line: -prtemca prte_allocation_required 1 # [ras-base:addhost-not-found] A request was made to add hosts from a hostfile, but the file could not be opened for reading: File: %s Please check the filename and try again. # [ras-base:nonuniform-slots] A request was made to add hosts from a hostfile while operating in a managed allocation. In this case, either the slots must be specified in the given hostfile, or the number of slots assigned by the resource manager on the existing nodes must be uniform. The current allocation does not conform to that requirement: Base number of slots: %d Node: %s Number of slots: %d Please assign a number of slots for each node to be added to the allocation. prrte-4.1.1~rc2/src/mca/ras/base/ras_base_close.c0000664000175000017500000000336715203542132022006 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/base/base.h" int prte_ras_base_finalize(void) { if (NULL != prte_ras_base.active_module) { prte_ras_base.active_module->finalize(); } return PRTE_SUCCESS; } int prte_ras_base_close(void) { /* Close all remaining available components (may be one if this is a PRTE program, or [possibly] multiple if this is ompi_info) */ mca_base_components_close(prte_ras_base_framework.framework_output, &prte_ras_base.ras_opened, NULL); /* Close the framework output */ pmix_output_close(prte_ras_base_framework.framework_output); prte_ras_base_framework.framework_output = -1; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/base/base.h0000664000175000017500000000464215203542132017756 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_RAS_BASE_H #define PRTE_MCA_RAS_BASE_H /* * includes */ #include "prte_config.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/mca/ras/ras.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_printf.h" /* * Global functions for MCA overall collective open and close */ BEGIN_C_DECLS /* * MCA Framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_ras_base_framework; /* select a component */ PRTE_EXPORT int prte_ras_base_select(void); /* * globals that might be needed */ typedef struct prte_ras_base_t { bool first_pass_completed; bool allocation_read; prte_ras_base_module_t *active_module; int total_slots_alloc; int multiplier; bool launch_orted_on_hn; bool simulated; } prte_ras_base_t; PRTE_EXPORT extern prte_ras_base_t prte_ras_base; /** * Add the specified node definitions to the registry */ PRTE_EXPORT int prte_ras_base_node_insert(pmix_list_t *, prte_job_t *); PRTE_EXPORT void prte_ras_base_display_alloc(prte_job_t *jdata); PRTE_EXPORT void prte_ras_base_display_cpus(prte_job_t *jdata, char *nodelist); PRTE_EXPORT void prte_ras_base_allocate(int fd, short args, void *cbdata); PRTE_EXPORT void prte_ras_base_modify(int fd, short args, void *cbdata); PRTE_EXPORT int prte_ras_base_add_hosts(prte_job_t *jdata); PRTE_EXPORT char *prte_ras_base_flag_string(prte_node_t *node); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/base/ras_base_allocate.c0000664000175000017500000013002715203542132022457 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include "constants.h" #include "types.h" #include "src/class/pmix_list.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/iof/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_argv.h" #include "src/util/dash_host/dash_host.h" #include "src/util/error_strings.h" #include "src/util/hostfile/hostfile.h" #include "src/util/name_fns.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_show_help.h" #include "src/util/pmix_string_copy.h" #include "src/util/proc_info.h" #include "src/util/prte_cmd_line.h" #include "src/mca/ras/base/base.h" char *prte_ras_base_flag_string(prte_node_t *node) { char *tmp, *t3, **t2 = NULL; if (0 == node->flags) { tmp = strdup("Flags: NONE"); return tmp; } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_DAEMON_LAUNCHED)) { PMIx_Argv_append_nosize(&t2, "DAEMON_LAUNCHED"); } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_LOC_VERIFIED)) { PMIx_Argv_append_nosize(&t2, "LOCATION_VERIFIED"); } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_OVERSUBSCRIBED)) { PMIx_Argv_append_nosize(&t2, "OVERSUBSCRIBED"); } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_MAPPED)) { PMIx_Argv_append_nosize(&t2, "MAPPED"); } if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { PMIx_Argv_append_nosize(&t2, "SLOTS_GIVEN"); } if (PRTE_FLAG_TEST(node, PRTE_NODE_NON_USABLE)) { PMIx_Argv_append_nosize(&t2, "NONUSABLE"); } if (NULL != t2) { t3 = PMIx_Argv_join(t2, ':'); pmix_asprintf(&tmp, "Flags: %s", t3); free(t3); PMIx_Argv_free(t2); } else { tmp = strdup("Flags: NONE"); } return tmp; } /* function to display allocation */ void prte_ras_base_display_alloc(prte_job_t *jdata) { char *tmp = NULL, *tmp2, *tmp3; int i, istart; prte_node_t *alloc; char *flgs, *aliases; bool parsable; pmix_proc_t source; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ALLOC_DISPLAYED, NULL, PMIX_BOOL)) { return; } parsable = prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT, NULL, PMIX_BOOL); PMIX_LOAD_PROCID(&source, jdata->nspace, PMIX_RANK_WILDCARD); if (parsable) { pmix_asprintf(&tmp, "\n"); } else { pmix_asprintf(&tmp, "\n====================== ALLOCATED NODES FOR JOB %s ======================\n", jdata->nspace); } if (prte_hnp_is_allocated) { istart = 0; } else { istart = 1; } for (i = istart; i < prte_node_pool->size; i++) { if (NULL == (alloc = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, i))) { continue; } if (parsable) { /* need to create the output in XML format */ pmix_asprintf(&tmp2, "\t\n", (NULL == alloc->name) ? "UNKNOWN" : alloc->name, (int) alloc->slots, (int) alloc->slots_max, (int) alloc->slots_inuse); } else { /* build the flags string */ flgs = prte_ras_base_flag_string(alloc); /* build the aliases string */ if (NULL != alloc->aliases) { aliases = PMIx_Argv_join(alloc->aliases, ','); } else { aliases = NULL; } pmix_asprintf(&tmp2, " %s: slots=%d max_slots=%d slots_inuse=%d state=%s\n\t%s\n\taliases: %s\n", (NULL == alloc->name) ? "UNKNOWN" : alloc->name, (int) alloc->slots, (int) alloc->slots_max, (int) alloc->slots_inuse, prte_node_state_to_str(alloc->state), flgs, (NULL == aliases) ? "NONE" : aliases); free(flgs); if (NULL != aliases) { free(aliases); } } if (NULL == tmp) { tmp = tmp2; } else { pmix_asprintf(&tmp3, "%s%s", tmp, tmp2); free(tmp); free(tmp2); tmp = tmp3; } } if (parsable) { pmix_asprintf(&tmp2, "%s\n", tmp); } else { pmix_asprintf(&tmp2, "%s=================================================================\n", tmp); } free(tmp); prte_iof_base_output(&source, PMIX_FWD_STDOUT_CHANNEL, tmp2); prte_set_attribute(&jdata->attributes, PRTE_JOB_ALLOC_DISPLAYED, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } static void display_cpus(prte_topology_t *t, prte_job_t *jdata, char *node) { char tmp[2048]; unsigned pkg, npkgs; bool bits_as_cores = false, use_hwthread_cpus = prte_hwloc_default_use_hwthread_cpus; unsigned npus, ncores; hwloc_obj_t obj; hwloc_cpuset_t avail = NULL; hwloc_cpuset_t allowed; hwloc_cpuset_t coreset = NULL; bool parsable; parsable = prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_PARSEABLE_OUTPUT, NULL, PMIX_BOOL); npus = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_PU); ncores = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_CORE); if (npus == ncores && !use_hwthread_cpus) { /* the bits in this bitmap represent cores */ bits_as_cores = true; } use_hwthread_cpus = prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL); if (!use_hwthread_cpus && !bits_as_cores) { coreset = hwloc_bitmap_alloc(); } avail = hwloc_bitmap_alloc(); if (parsable) { pmix_output(prte_clean_output, "", node); } else { pmix_output(prte_clean_output, "\n====================== AVAILABLE PROCESSORS [node: %s] ======================\n\n", node); } npkgs = prte_hwloc_base_get_nbobjs_by_type(t->topo, HWLOC_OBJ_PACKAGE); allowed = (hwloc_cpuset_t)hwloc_topology_get_allowed_cpuset(t->topo); for (pkg = 0; pkg < npkgs; pkg++) { obj = prte_hwloc_base_get_obj_by_type(t->topo, HWLOC_OBJ_PACKAGE, pkg); hwloc_bitmap_and(avail, obj->cpuset, allowed); if (hwloc_bitmap_iszero(avail)) { if (parsable) { pmix_output(prte_clean_output, " ", pkg, "NONE"); } else { pmix_output(prte_clean_output, "PKG[%d]: NONE", pkg); } continue; } if (bits_as_cores) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); if (parsable) { pmix_output(prte_clean_output, " ", pkg, tmp); } else { pmix_output(prte_clean_output, "PKG[%d]: %s", pkg, tmp); } } else if (use_hwthread_cpus) { /* can just use the hwloc fn directly */ hwloc_bitmap_list_snprintf(tmp, 2048, avail); if (parsable) { pmix_output(prte_clean_output, " ", pkg, tmp); } else { pmix_output(prte_clean_output, "PKG[%d]: %s", pkg, tmp); } } else { prte_hwloc_build_map(t->topo, avail, use_hwthread_cpus | bits_as_cores, coreset); /* now print out the string */ hwloc_bitmap_list_snprintf(tmp, 2048, coreset); if (parsable) { pmix_output(prte_clean_output, " ", pkg, tmp); } else { pmix_output(prte_clean_output, "PKG[%d]: %s", pkg, tmp); } } } hwloc_bitmap_free(avail); if (NULL != coreset) { hwloc_bitmap_free(coreset); } if (parsable) { pmix_output(prte_clean_output, "\n"); } else { pmix_output(prte_clean_output, "\n======================================================================\n"); } return; } void prte_ras_base_display_cpus(prte_job_t *jdata, char *nodelist) { char **nodes = NULL; int i, j, m; prte_topology_t *t; prte_node_t *nptr; bool moveon; if (NULL == nodelist) { /* output the available cpus for all topologies */ for (i=0; i < prte_node_topologies->size; i++) { t = (prte_topology_t*)pmix_pointer_array_get_item(prte_node_topologies, i); if (NULL != t) { display_cpus(t, jdata, "N/A"); } } return; } nodes = PMIx_Argv_split(nodelist, ';'); for (j=0; NULL != nodes[j]; j++) { moveon = false; for (i=0; i < prte_node_pool->size && !moveon; i++) { nptr = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, i); if (NULL == nptr) { continue; } if (0 == strcmp(nptr->name, nodes[j])) { display_cpus(nptr->topology, jdata, nodes[j]); break; } if (NULL == nptr->aliases) { continue; } /* no choice but an exhaustive search - fortunately, these lists are short! */ for (m = 0; NULL != nptr->aliases[m]; m++) { if (0 == strcmp(nodes[j], nptr->aliases[m])) { /* this is the node! */ display_cpus(nptr->topology, jdata, nodes[j]); moveon = true; break; } } } } PMIx_Argv_free(nodes); } /* * Function for selecting one component from all those that are * available. */ void prte_ras_base_allocate(int fd, short args, void *cbdata) { int rc; prte_job_t *jdata; pmix_list_t nodes; prte_node_t *node; int32_t i, j; prte_app_context_t *app; prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; char *hosts = NULL, **hostlist; char *ptr; pmix_status_t ret; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* convenience */ jdata = caddy->jdata; /* construct a list to hold the results */ PMIX_CONSTRUCT(&nodes, pmix_list_t); if (prte_ras_base.simulated) { prte_set_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } /* if we already did this, don't do it again - the pool of * global resources is set. */ if (prte_ras_base.first_pass_completed) { if (!prte_managed_allocation && !prte_ras_base.allocation_read) { // unmanaged system and no prior allocation read - check hostfiles goto parsehostfile; } // already read an allocation PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate allocation already read", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_DESTRUCT(&nodes); goto next_state; } prte_ras_base.first_pass_completed = true; /* Otherwise, we have to create * the initial set of resources that will delineate all * further operations serviced by this HNP. This list will * contain ALL nodes that can be used by any subsequent job. * * In other words, if a node isn't found in this step, then * no job launched by this HNP will be able to utilize it. */ /* if a component was selected, then we know we are in a managed * environment. - the active module will return a list of what it found */ if (NULL != prte_ras_base.active_module) { /* read the allocation */ if (PRTE_SUCCESS != (rc = prte_ras_base.active_module->allocate(jdata, &nodes))) { if (PRTE_ERR_ALLOCATION_PENDING == rc) { /* an allocation request is underway, so just do nothing */ PMIX_DESTRUCT(&nodes); PMIX_RELEASE(caddy); return; } if (PRTE_ERR_SYSTEM_WILL_BOOTSTRAP == rc) { /* this module indicates that nodes will be discovered * on a bootstrap basis, so all we do here is add our * own node to the list */ goto addlocal; } if (PRTE_ERR_TAKE_NEXT_OPTION == rc) { /* we have an active module, but it is unable to * allocate anything for this job - this indicates * that it isn't a fatal error, but could be if * an allocation is required */ if (prte_allocation_required) { /* an allocation is required, so this is fatal */ PMIX_DESTRUCT(&nodes); pmix_show_help("help-ras-base.txt", "ras-base:no-allocation", true); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } else { /* an allocation is not required, so we can just * run on the local node - go add it */ goto addlocal; } } PRTE_ERROR_LOG(rc); PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } } /* If something came back, save it and we are done */ if (!pmix_list_is_empty(&nodes)) { /* flag that the allocation is managed */ prte_managed_allocation = true; /* since this is a managed allocation, we do not resolve */ prte_do_not_resolve = true; // mark that we received an allocation prte_ras_base.allocation_read = true; /* if we are not retaining FQDN hostnames, then record * aliases where appropriate */ PMIX_LIST_FOREACH(node, &nodes, prte_node_t) { if (!pmix_net_isaddr(node->name) && NULL != (ptr = strchr(node->name, '.'))) { node->rawname = strdup(node->name); if (prte_keep_fqdn_hostnames) { /* retain the non-fqdn name as an alias */ *ptr = '\0'; PMIx_Argv_append_unique_nosize(&node->aliases, node->name); *ptr = '.'; } else { /* add the fqdn name as an alias */ PMIx_Argv_append_unique_nosize(&node->aliases, node->name); /* retain the non-fqdn name as the node's name */ *ptr = '\0'; } } } /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } PMIX_DESTRUCT(&nodes); goto DISPLAY; } else if (prte_allocation_required) { /* if nothing was found, and an allocation is * required, then error out */ PMIX_DESTRUCT(&nodes); pmix_show_help("help-ras-base.txt", "ras-base:no-allocation", true); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } parsehostfile: PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate nothing found in module - proceeding to hostfile", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* nothing was found, or no active module was alive. We first see * if we were given a rank/seqfile - if so, use it as the hosts will be * taken from the mapping */ hosts = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_FILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate parsing rank/seqfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); /* a rank/seqfile was provided - parse it */ if (PRTE_SUCCESS != (rc = prte_util_add_hostfile_nodes(&nodes, hosts))) { PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); free(hosts); return; } free(hosts); } /* if something was found in the rankfile, we use that as our global * pool - set it and we are done */ if (!pmix_list_is_empty(&nodes)) { // mark that we received an allocation prte_ras_base.allocation_read = true; /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } /* Record that the rankfile mapping policy has been selected */ PRTE_SET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping, PRTE_MAPPING_GIVEN); PRTE_SET_MAPPING_POLICY(prte_rmaps_base.mapping, PRTE_MAPPING_BYUSER); /* rankfile is considered equivalent to an RM allocation */ if (!(PRTE_MAPPING_SUBSCRIBE_GIVEN & PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping))) { PRTE_SET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping, PRTE_MAPPING_NO_OVERSUBSCRIBE); } /* cleanup */ PMIX_DESTRUCT(&nodes); goto DISPLAY; } /* if a dash-host has been provided, aggregate across all the * app_contexts. Any hosts the user wants to add via comm_spawn * can be done so using the add_host option */ for (i = 0; i < jdata->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i))) { continue; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_DASH_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate adding dash_hosts", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (PRTE_SUCCESS != (rc = prte_util_add_dash_host_nodes(&nodes, hosts, true))) { free(hosts); PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } free(hosts); } } /* if something was found in the dash-host(s), we use that as our global * pool - set it and we are done */ if (!pmix_list_is_empty(&nodes)) { // mark that we received an allocation prte_ras_base.allocation_read = true; /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } /* cleanup */ PMIX_DESTRUCT(&nodes); goto DISPLAY; } /* Our next option is to look for a hostfile and assign our global * pool from there. * * Individual hostfile names, if given, are included * in the app_contexts for this job. We therefore need to * retrieve the app_contexts for the job, and then cycle * through them to see if anything is there. The parser will * add the nodes found in each hostfile to our list - i.e., * the resulting list contains the UNION of all nodes specified * in hostfiles from across all app_contexts * * Note that any relative node syntax found in the hostfiles will * generate an error in this scenario, so only non-relative syntax * can be present */ for (i = 0; i < jdata->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i))) { continue; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_HOSTFILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate adding hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); /* hostfile was specified - parse it and add it to the list */ hostlist = PMIx_Argv_split(hosts, ','); free(hosts); for (j=0; NULL != hostlist[j]; j++) { if (PRTE_SUCCESS != (rc = prte_util_add_hostfile_nodes(&nodes, hostlist[j]))) { PMIx_Argv_free(hostlist); PMIX_DESTRUCT(&nodes); /* set an error event */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } } PMIx_Argv_free(hostlist); } } /* if something was found in the hostfiles(s), we use that as our global * pool - set it and we are done */ if (!pmix_list_is_empty(&nodes)) { // mark that we received an allocation prte_ras_base.allocation_read = true; /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } /* cleanup */ PMIX_DESTRUCT(&nodes); goto DISPLAY; } /* if nothing was found so far, then look for a default hostfile */ if (NULL != prte_default_hostfile) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate parsing default hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), prte_default_hostfile)); /* a default hostfile was provided - parse it */ if (PRTE_SUCCESS != (rc = prte_util_add_hostfile_nodes(&nodes, prte_default_hostfile))) { PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } } /* if something was found in the default hostfile, we use that as our global * pool - set it and we are done */ if (!pmix_list_is_empty(&nodes)) { // mark that we received an allocation prte_ras_base.allocation_read = true; /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } /* cleanup */ PMIX_DESTRUCT(&nodes); goto DISPLAY; } PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:allocate nothing found in hostfiles - inserting current node", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); addlocal: /* if nothing was found by any of the above methods, then we have no * earthly idea what to do - so just add the local host */ node = PMIX_NEW(prte_node_t); if (NULL == node) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } /* use the same name we got in prte_process_info so we avoid confusion in * the session directories */ node->name = strdup(prte_process_info.nodename); node->state = PRTE_NODE_STATE_UP; node->slots_inuse = 0; node->slots_max = 0; node->slots = 1; pmix_list_append(&nodes, &node->super); /* mark the HNP as "allocated" since we have nothing else to use */ prte_hnp_is_allocated = true; /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); PMIX_DESTRUCT(&nodes); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); return; } PMIX_DESTRUCT(&nodes); DISPLAY: /* shall we display the results? */ if (4 < pmix_output_get_verbosity(prte_ras_base_framework.framework_output)) { prte_ras_base_display_alloc(jdata); } next_state: /* are we to report this event? */ if (prte_report_events) { pmix_info_t info; PMIX_INFO_LOAD(&info, "prte.notify.donotloop", NULL, PMIX_BOOL); ret = PMIx_Notify_event(PMIX_NOTIFY_ALLOC_COMPLETE, NULL, PMIX_GLOBAL, &info, 1, NULL, NULL); if (PMIX_SUCCESS != ret && PMIX_OPERATION_SUCCEEDED != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOC_FAILED); PMIX_RELEASE(caddy); } PMIX_INFO_DESTRUCT(&info); } /* set total slots alloc */ jdata->total_slots_alloc = prte_ras_base.total_slots_alloc; hosts = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_TOPO, (void**)&hosts, PMIX_STRING)) { if (NULL != hosts) { hostlist = PMIx_Argv_split(hosts, ';'); free(hosts); for (j=0; NULL != hostlist[j]; j++) { node = prte_node_match(NULL, hostlist[j]); if (NULL == node) { continue; } pmix_output(prte_clean_output, "=================================================================\n"); pmix_output(prte_clean_output, "TOPOLOGY FOR NODE %s", node->name); prte_hwloc_print(&ptr, NULL, node->topology->topo); pmix_output(prte_clean_output, "%s", ptr); free(ptr); pmix_output(prte_clean_output, "=================================================================\n"); } PMIx_Argv_free(hostlist); } else { for (j=0; j < prte_node_pool->size; j++) { node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, j); if (NULL == node) { continue; } pmix_output(prte_clean_output, "=================================================================\n"); pmix_output(prte_clean_output, "TOPOLOGY FOR NODE %s", node->name); prte_hwloc_print(&ptr, NULL, node->topology->topo); pmix_output(prte_clean_output, "%s", ptr); free(ptr); pmix_output(prte_clean_output, "=================================================================\n"); } } } /* set the job state to the next position */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALLOCATION_COMPLETE); /* cleanup */ PMIX_RELEASE(caddy); } static void localrelease(void *cbdata) { prte_pmix_server_req_t *req = (prte_pmix_server_req_t*)cbdata; pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL); PMIX_RELEASE(req); } void prte_ras_base_modify(int fd, short args, void *cbdata) { prte_pmix_server_req_t *req = (prte_pmix_server_req_t*)cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); if (NULL != prte_ras_base.active_module && NULL != prte_ras_base.active_module->modify) { prte_ras_base.active_module->modify(req); } else { req->pstatus = PMIX_ERR_NOT_SUPPORTED; } // execute the callback if (NULL != req->infocbfunc) { req->infocbfunc(req->pstatus, req->info, req->ninfo, req->cbdata, localrelease, req); return; } pmix_pointer_array_set_item(&prte_pmix_server_globals.local_reqs, req->local_index, NULL); PMIX_RELEASE(req); } int prte_ras_base_add_hosts(prte_job_t *jdata) { int rc; pmix_list_t nodes; int i, k, m, n, slots; prte_app_context_t *app; prte_node_t *node, *next, *nptr; char *hosts, *line, *cptr, *ptr, **hostfiles, *nm; FILE *fp; bool addslots, found; bool extend = false; int default_slots = -1; PMIX_CONSTRUCT(&nodes, pmix_list_t); // see if we have any add-hostfile or add-host directives for (i = 0; i < jdata->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i))) { continue; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_ADD_HOSTFILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { // found one free(hosts); goto proceed; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_ADD_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { // found one free(hosts); goto proceed; } } // if we get here, then there were no directives return PRTE_SUCCESS; proceed: /* Individual add-hostfile names, if given, are included * in the app_contexts for this job. We therefore need to * retrieve the app_contexts for the job, and then cycle * through them to see if anything is there. The parser will * add the nodes found in each add-hostfile to our list - i.e., * the resulting list contains the UNION of all nodes specified * in add-hostfiles from across all app_contexts * * Note that any relative node syntax found in the add-hostfiles will * generate an error in this scenario, so only non-relative syntax * can be present */ /* if we are in a managed allocation, the best we can do for nodes * that do not include a specific slot assignment is to (a) check * to see if there is a uniform assignment on existing nodes and * use that, or (b) generate an error as we cannot know what the * host environment might have set */ if (prte_managed_allocation) { for (n = 0; n < prte_node_pool->size; n++) { nptr = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, n); if (NULL == nptr) { continue; } if (-1 == default_slots) { default_slots = nptr->slots; continue; } if (default_slots != nptr->slots) { // generate an error message pmix_show_help("help-ras-base.txt", "ras-base:nonuniform-slots", true, default_slots, nptr->name, nptr->slots); PMIX_LIST_DESTRUCT(&nodes); return PRTE_ERR_SILENT; } } } for (i = 0; i < jdata->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i))) { continue; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_ADD_HOSTFILE, (void **) &hosts, PMIX_STRING) && NULL != hosts) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:add_hosts checking add-hostfile %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts)); prte_remove_attribute(&app->attributes, PRTE_APP_ADD_HOSTFILE); hostfiles = PMIx_Argv_split(hosts, ','); free(hosts); for (k=0; NULL != hostfiles[k]; k++) { /* hostfile was specified - parse it and add it to the list. We * don't use the hostfile parsing code in src/util because it * uses flex and that has problems handling the range of allowed * syntax here */ fp = fopen(hostfiles[k], "r"); if (NULL == fp) { pmix_show_help("help-ras-base.txt", "ras-base:addhost-not-found", true, hostfiles[k]); PMIx_Argv_free(hostfiles); PMIX_LIST_DESTRUCT(&nodes); return PRTE_ERR_SILENT; } while (NULL != (line = pmix_getline(fp))) { // ignore comments and blank lines if (0 == strlen(line)) { free(line); continue; } // remove leading whitespace cptr = line; while (isspace(*cptr)) { ++cptr; } if ('#' == *cptr) { free(line); continue; } addslots = false; // because there can be arbitrary whitespace around keywords, // we manually parse the line to get the directives ptr = cptr; while ('\0' != *ptr && !isspace(*ptr)) { ++ptr; } if ('\0' == *ptr) { // end of the line - just the node name was given slots = default_slots; goto process; } *ptr = '\0'; // terminate the name // find the '=' sign ++ptr; while ('\0' != *ptr && ('=' != *ptr || isspace(*ptr))) { ++ptr; } if ('\0' == *ptr) { // didn't specify slots - use the default value slots = default_slots; goto process; } // find the value ++ptr; while ('\0' != *ptr && isspace(*ptr)) { ++ptr; } if ('\0' == *ptr) { // bad syntax PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); fclose(fp); free(line); PMIx_Argv_free(hostfiles); PMIX_LIST_DESTRUCT(&nodes); return PRTE_ERR_SILENT; } // if it is a '+' or '-', then we are adjusting // the #slots if ('+' == *ptr || '-' == *ptr) { addslots = true; } slots = strtol(ptr, NULL, 10); process: // see if we have this node found = false; // does the name refer to me? if (prte_check_host_is_local(cptr)) { nm = prte_process_info.nodename; } else { nm = cptr; } for (n = 0; !found && n < prte_node_pool->size; n++) { nptr = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, n); if (NULL == nptr) { continue; } if (0 == strcmp(nm, nptr->name)) { // we have the node if (addslots) { nptr->slots += slots; if (0 > nptr->slots) { nptr->slots = 0; } } found = true; break; } else if (NULL != nptr->aliases) { /* no choice but an exhaustive search - fortunately, these lists are short! */ for (m = 0; NULL != nptr->aliases[m]; m++) { if (0 == strcmp(cptr, nptr->aliases[m])) { if (addslots) { nptr->slots += slots; if (0 > nptr->slots) { nptr->slots = 0; } } found = true; break; } } } } if (!found) { // this is a new node - add it node = PMIX_NEW(prte_node_t); node->name = strdup(cptr); node->slots = slots; node->state = PRTE_NODE_STATE_ADDED; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); pmix_list_append(&nodes, &node->super); } free(line); } fclose(fp); } PMIx_Argv_free(hostfiles); } } if (!pmix_list_is_empty(&nodes)) { /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); } /* mark that an updated nidmap must be communicated to existing daemons */ prte_nidmap_communicated = false; extend = true; } /* We next check for and add any add-host options. Note this is * a -little- different than dash-host in that (a) we add these * nodes to the global pool (avoiding duplication), * and (b) as a result, any job and/or app_context can access them. * * Note that any relative node syntax found in the add-host lists will * generate an error in this scenario, so only non-relative syntax * can be present */ for (i = 0; i < jdata->apps->size; i++) { if (NULL == (app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, i))) { continue; } hosts = NULL; if (prte_get_attribute(&app->attributes, PRTE_APP_ADD_HOST, (void **) &hosts, PMIX_STRING) && NULL != hosts) { pmix_output_verbose(5, prte_ras_base_framework.framework_output, "%s ras:base:add_hosts checking add-host %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), hosts); if (PRTE_SUCCESS != (rc = prte_util_add_dash_host_nodes(&nodes, hosts, true))) { PRTE_ERROR_LOG(rc); PMIX_DESTRUCT(&nodes); free(hosts); return rc; } prte_remove_attribute(&app->attributes, PRTE_APP_ADD_HOST); free(hosts); } } /* if something was found, we add that to our global pool */ if (!pmix_list_is_empty(&nodes)) { /* the node insert code doesn't check for uniqueness, so we will * do so here - yes, this is an ugly, non-scalable loop, but this * is the exception case and so we can do it here */ PMIX_LIST_FOREACH_SAFE(node, next, &nodes, prte_node_t) { node->state = PRTE_NODE_STATE_ADDED; found = false; for (n = 0; !found && n < prte_node_pool->size; n++) { nptr = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, n); if (NULL == nptr) { continue; } if (0 == strcmp(node->name, nptr->name)) { if (prte_get_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS, NULL, PMIX_BOOL)) { nptr->slots += node->slots; prte_remove_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS); } else { nptr->slots = node->slots; } pmix_list_remove_item(&nodes, &node->super); PMIX_RELEASE(node); found = true; } else if (NULL != nptr->aliases) { /* no choice but an exhaustive search - fortunately, these lists are short! */ for (m = 0; !found && NULL != nptr->aliases[m]; m++) { if (0 == strcmp(node->name, nptr->aliases[m])) { if (prte_get_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS, NULL, PMIX_BOOL)) { nptr->slots += node->slots; prte_remove_attribute(&node->attributes, PRTE_NODE_ADD_SLOTS); } else { nptr->slots = node->slots; } pmix_list_remove_item(&nodes, &node->super); PMIX_RELEASE(node); found = true; } } } } } if (!pmix_list_is_empty(&nodes)) { /* store the results in the global resource pool - this removes the * list items */ if (PRTE_SUCCESS != (rc = prte_ras_base_node_insert(&nodes, jdata))) { PRTE_ERROR_LOG(rc); } /* mark that an updated nidmap must be communicated to existing daemons */ prte_nidmap_communicated = false; extend = true; } } /* cleanup */ PMIX_LIST_DESTRUCT(&nodes); if (extend) { // mark that we need to extend the DVM prte_set_attribute(&jdata->attributes, PRTE_JOB_EXTEND_DVM, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } /* shall we display the results? */ if (0 < pmix_output_get_verbosity(prte_ras_base_framework.framework_output) || prte_get_attribute(&jdata->attributes, PRTE_JOB_DISPLAY_ALLOC, NULL, PMIX_BOOL)) { prte_ras_base_display_alloc(jdata); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/base/Makefile.am0000664000175000017500000000216615203542132020726 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ base/help-ras-base.txt headers += \ base/base.h libprtemca_ras_la_SOURCES += \ base/ras_base_frame.c \ base/ras_base_select.c \ base/ras_base_allocate.c \ base/ras_base_node.c prrte-4.1.1~rc2/src/mca/ras/base/ras_base_select.c0000664000175000017500000000440215203542132022147 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/ras/base/base.h" /* * Select one RAS component from all those that are available. */ int prte_ras_base_select(void) { /* For all other systems, provide the following support */ prte_ras_base_component_t *best_component = NULL; prte_ras_base_module_t *best_module = NULL; /* * Select the best component */ if (PRTE_SUCCESS != pmix_mca_base_select("ras", prte_ras_base_framework.framework_output, &prte_ras_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL)) { /* This will only happen if no component was selected */ /* If we didn't find one to select, that is okay */ return PRTE_SUCCESS; } /* Save the winner */ /* No component saved */ prte_ras_base.active_module = best_module; if (NULL != prte_ras_base.active_module->init) { return prte_ras_base.active_module->init(); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/base/ras_base_node.c0000664000175000017500000002243015203542132021616 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/util/pmix_argv.h" #include "src/util/pmix_if.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/base/base.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/pmix_net.h" #include "src/util/proc_info.h" #include "src/mca/ras/base/base.h" /* * Add the specified node definitions to the global data store * NOTE: this removes all items from the list! */ int prte_ras_base_node_insert(pmix_list_t *nodes, prte_job_t *jdata) { pmix_list_item_t *item; int32_t num_nodes; int rc, i; prte_node_t *node, *hnp_node, *nptr; bool skiphnp = false; prte_attribute_t *kv; prte_proc_t *daemon; prte_job_t *djob; /* get the number of nodes */ num_nodes = (int32_t) pmix_list_get_size(nodes); if (0 == num_nodes) { return PRTE_SUCCESS; /* nothing to do */ } PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:node_insert inserting %ld nodes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (long) num_nodes)); /* mark the job as being a large-cluster sim if that was requested */ if (1 < prte_ras_base.multiplier && NULL != jdata) { prte_set_attribute(&jdata->attributes, PRTE_JOB_MULTI_DAEMON_SIM, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } /* set the size of the global array - this helps minimize time * spent doing realloc's */ rc = pmix_pointer_array_set_size(prte_node_pool, num_nodes * prte_ras_base.multiplier); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); return rc; } /* if we are not launching, get the daemon job */ djob = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* get the hnp node's info */ hnp_node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, 0); if (prte_ras_base.launch_orted_on_hn) { if (NULL != hnp_node) { PMIX_LIST_FOREACH(node, nodes, prte_node_t) { if (prte_check_host_is_local(node->name)) { prte_hnp_is_allocated = true; break; } } if (prte_hnp_is_allocated && !(PRTE_GET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping) & PRTE_MAPPING_NO_USE_LOCAL)) { if (NULL != hnp_node->name) { free(hnp_node->name); } hnp_node->name = strdup("prte"); skiphnp = true; PRTE_SET_MAPPING_DIRECTIVE(prte_rmaps_base.mapping, PRTE_MAPPING_NO_USE_LOCAL); PRTE_FLAG_SET(hnp_node, PRTE_NODE_NON_USABLE); // leave this node out of mapping operations } } } /* cycle through the list */ while (NULL != (item = pmix_list_remove_first(nodes))) { node = (prte_node_t *) item; /* the HNP had to already enter its node on the array - that entry is in the * first position since it is the first one entered. We need to check to see * if this node is the same as the HNP's node so we don't double-enter it */ if (!skiphnp && NULL != hnp_node && prte_check_host_is_local(node->name)) { PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:node_insert updating HNP [%s] info to %ld slots", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, (long) node->slots)); /* flag that hnp has been allocated */ prte_hnp_is_allocated = true; /* update the total slots in the job */ prte_ras_base.total_slots_alloc += node->slots; /* copy the allocation data to that node's info */ hnp_node->slots = node->slots; hnp_node->slots_max = node->slots_max; /* copy across any attributes */ PMIX_LIST_FOREACH(kv, &node->attributes, prte_attribute_t) { prte_set_attribute(&node->attributes, kv->key, PRTE_ATTR_LOCAL, &kv->data, kv->data.type); } if (prte_managed_allocation || PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { /* the slots are always treated as sacred * in managed allocations */ PRTE_FLAG_SET(hnp_node, PRTE_NODE_FLAG_SLOTS_GIVEN); } else { PRTE_FLAG_UNSET(hnp_node, PRTE_NODE_FLAG_SLOTS_GIVEN); } /* if the node name is different, store it as an alias */ if (0 != strcmp(hnp_node->name, node->name)) { PMIx_Argv_append_unique_nosize(&hnp_node->aliases, node->name); } // transfer across any new aliases if (NULL != node->aliases) { for (i=0; NULL != node->aliases[i]; i++) { PMIx_Argv_append_unique_nosize(&hnp_node->aliases, node->aliases[i]); } } if (NULL != node->rawname) { if (NULL != hnp_node->rawname) { free(hnp_node->rawname); } hnp_node->rawname = strdup(node->rawname); } /* don't keep duplicate copy */ PMIX_RELEASE(node); /* create copies, if required */ for (i = 1; i < prte_ras_base.multiplier; i++) { rc = prte_node_copy(&node, hnp_node); if (PRTE_SUCCESS != rc) { return rc; } PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_DAEMON_LAUNCHED); node->index = pmix_pointer_array_add(prte_node_pool, node); } } else { /* insert the object onto the prte_nodes global array */ PMIX_OUTPUT_VERBOSE((5, prte_ras_base_framework.framework_output, "%s ras:base:node_insert node %s slots %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == node->name) ? "NULL" : node->name, node->slots)); if (prte_managed_allocation) { /* the slots are always treated as sacred * in managed allocations */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } /* insert it into the array */ node->index = pmix_pointer_array_add(prte_node_pool, (void *) node); if (PRTE_SUCCESS > (rc = node->index)) { PRTE_ERROR_LOG(rc); return rc; } if (prte_get_attribute(&djob->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* create a daemon for this node since we won't be launching * and the mapper needs to see a daemon - this is used solely * for testing the mappers */ daemon = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&daemon->name, PRTE_PROC_MY_NAME->nspace, node->index); daemon->state = PRTE_PROC_STATE_RUNNING; PMIX_RETAIN(node); daemon->node = node; pmix_pointer_array_set_item(djob->procs, daemon->name.rank, daemon); djob->num_procs++; PMIX_RETAIN(daemon); node->daemon = daemon; } /* update the total slots in the job */ prte_ras_base.total_slots_alloc += node->slots; /* check if we have fqdn names in the allocation */ if (!pmix_net_isaddr(node->name) && NULL != strchr(node->name, '.')) { prte_have_fqdn_allocation = true; } /* duplicate the node if requested */ for (i = 1; i < prte_ras_base.multiplier; i++) { rc = prte_node_copy(&nptr, node); if (PRTE_SUCCESS != rc) { return rc; } nptr->index = pmix_pointer_array_add(prte_node_pool, nptr); } } } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/ras.h0000664000175000017500000001110115203542132016703 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The PRTE Resource Allocation Subsystem (RAS) * * The resource allocation subsystem is responsible for determining * what (if any) resources have been allocated to the specified job * (via some prior action), and to obtain an allocation (if possible) * if resources have NOT been previously allocated. It is anticipated * that PRTE users will execute an "mpirun" or other command that * invokes PRTE through one of two channels: * * 1. the user will login to the computing resource they intend * to use, request a resource allocation from that system, and then * execute the mpirun or other command. Thus, the allocation has * already been obtained prior to PRTE's initialization. In most * cases, systems pass allocation information via environmental * parameters. Thus, the RAS components must know the correct * environmental parameter to look for within the environment they * seek to support (e.g., an LSF component should know that LSF passes * allocation parameters as a specific LSF-named entity). * * 2. the user issues an mpirun command or an application that uses * PRTE without obtaining an allocation in advance. Thus, the associated * RAS component must know how to request an allocation from the * designated resource. If it doesn't, or it cannot obtain the allocation, * then it shall indicate this by setting the system to an appropriate * state. */ #ifndef PRTE_MCA_RAS_H #define PRTE_MCA_RAS_H #include "prte_config.h" #include "constants.h" #include "types.h" #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS /* allocation event - the event one activates to schedule resource * allocation for pending jobs */ PRTE_EXPORT extern prte_event_t prte_allocate_event; /* * ras module functions - these are not accessible to the outside world, * but are defined here by convention */ /* init the module */ typedef int (*prte_ras_base_module_init_fn_t)(void); /** * Allocate resources to a job. */ typedef int (*prte_ras_base_module_allocate_fn_t)(prte_job_t *jdata, pmix_list_t *nodes); /* deallocate resources */ typedef void (*prte_ras_base_module_dealloc_fn_t)(prte_job_t *jdata, prte_app_context_t *app); /* modify allocation */ typedef void (*prte_ras_base_module_modify_fn_t)(prte_pmix_server_req_t *req); /** * Cleanup module resources. */ typedef int (*prte_ras_base_module_finalize_fn_t)(void); /** * ras module */ struct prte_ras_base_module_2_0_0_t { /** init */ prte_ras_base_module_init_fn_t init; /** Allocation function pointer */ prte_ras_base_module_allocate_fn_t allocate; prte_ras_base_module_dealloc_fn_t deallocate; prte_ras_base_module_modify_fn_t modify; /** Finalization function pointer */ prte_ras_base_module_finalize_fn_t finalize; }; /** Convenience typedef */ typedef struct prte_ras_base_module_2_0_0_t prte_ras_base_module_2_0_0_t; /** Convenience typedef */ typedef prte_ras_base_module_2_0_0_t prte_ras_base_module_t; /* * ras component */ /** Convenience typedef */ typedef pmix_mca_base_component_t prte_ras_base_component_t; /** * Macro for use in components that are of type ras */ #define PRTE_RAS_BASE_VERSION_2_0_0 PRTE_MCA_BASE_VERSION_3_0_0("ras", 2, 0, 0) END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/Makefile.am0000664000175000017500000000261415203542132020012 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2015 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_ras.la libprtemca_ras_la_SOURCES = # pkgdata setup EXTRA_DIST = # local files headers = ras.h libprtemca_ras_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/ras/gridengine/0000775000175000017500000000000015203542132020066 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/gridengine/ras_gridengine.h0000664000175000017500000000345415203542132023225 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * Resource allocation for Grid Engine */ #ifndef PRTE_RAS_GRIDENGINE_H #define PRTE_RAS_GRIDENGINE_H #include "prte_config.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS /** * RAS Component */ struct prte_ras_base_component_t { prte_ras_base_component_t super; int verbose; int priority; bool show_jobid; }; typedef struct prte_ras_base_component_t prte_mca_ras_gridengine_component_t; PRTE_EXPORT extern prte_mca_ras_gridengine_component_t prte_mca_ras_gridengine_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_gridengine_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/gridengine/owner.txt0000664000175000017500000000024515203542132021762 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: unmaintained prrte-4.1.1~rc2/src/mca/ras/gridengine/ras_gridengine_component.c0000664000175000017500000001205015203542132025272 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * Resource allocation for Grid Engine */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/util/pmix_output.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "ras_gridengine.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int prte_ras_gridengine_register(void); static int prte_ras_gridengine_open(void); static int prte_ras_gridengine_close(void); static int prte_mca_ras_gridengine_component_query(pmix_mca_base_module_t **module, int *priority); static int prte_ras_gridengine_verbose; prte_mca_ras_gridengine_component_t prte_mca_ras_gridengine_component = { .super = { PRTE_RAS_BASE_VERSION_2_0_0, .pmix_mca_component_name = "gridengine", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_open_component = prte_ras_gridengine_open, .pmix_mca_close_component = prte_ras_gridengine_close, .pmix_mca_query_component = prte_mca_ras_gridengine_component_query, .pmix_mca_register_component_params = prte_ras_gridengine_register, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, gridengine) static int prte_ras_gridengine_register(void) { pmix_mca_base_component_t *c = &prte_mca_ras_gridengine_component.super; prte_mca_ras_gridengine_component.priority = 100; (void) pmix_mca_base_component_var_register(c, "priority", "Priority of the gridengine ras component", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_mca_ras_gridengine_component.priority); prte_ras_gridengine_verbose = 0; (void) pmix_mca_base_component_var_register( c, "verbose", "Enable verbose output for the gridengine ras component", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_ras_gridengine_verbose); prte_mca_ras_gridengine_component.show_jobid = false; (void) pmix_mca_base_component_var_register(c, "show_jobid", "Show the JOB_ID of the Grid Engine job", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_mca_ras_gridengine_component.show_jobid); return PRTE_SUCCESS; } /** * component open/close/init function */ static int prte_ras_gridengine_open(void) { if (prte_ras_gridengine_verbose != 0) { prte_mca_ras_gridengine_component.verbose = pmix_output_open(NULL); } else { prte_mca_ras_gridengine_component.verbose = -1; } return PRTE_SUCCESS; } static int prte_mca_ras_gridengine_component_query(pmix_mca_base_module_t **module, int *priority) { *priority = prte_mca_ras_gridengine_component.priority; if (NULL != getenv("SGE_ROOT") && NULL != getenv("ARC") && NULL != getenv("PE_HOSTFILE") && NULL != getenv("JOB_ID")) { PMIX_OUTPUT_VERBOSE((2, prte_ras_base_framework.framework_output, "%s ras:gridengine: available for selection", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); *module = (pmix_mca_base_module_t *) &prte_ras_gridengine_module; return PRTE_SUCCESS; } PMIX_OUTPUT_VERBOSE((2, prte_ras_base_framework.framework_output, "%s ras:gridengine: NOT available for selection", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); *module = NULL; return PRTE_ERROR; } /** * Close all subsystems. */ static int prte_ras_gridengine_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/gridengine/configure.m40000664000175000017500000000317615203542132022320 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ras_sge_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ras_gridengine_CONFIG],[ AC_CONFIG_FILES([src/mca/ras/gridengine/Makefile]) PRTE_CHECK_GRIDENGINE([ras_gridengine], [ras_gridengine_happy="yes"], [ras_gridengine_happy="no"]) AS_IF([test "$ras_gridengine_happy" = "yes" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) ])dnl prrte-4.1.1~rc2/src/mca/ras/gridengine/ras_gridengine_module.c0000664000175000017500000001300015203542132024551 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file: * Resource Allocation for Grid Engine */ #include "prte_config.h" #include "constants.h" #include #include #include #include "src/mca/errmgr/errmgr.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/gridengine/ras_gridengine.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_net.h" #include "src/util/pmix_output.h" #include "src/util/pmix_show_help.h" /* * Local functions */ static int prte_ras_gridengine_allocate(prte_job_t *jdata, pmix_list_t *nodes); static int prte_ras_gridengine_finalize(void); #if 0 static int get_slot_count(char* node_name, int* slot_cnt); #endif /* * Global variable */ prte_ras_base_module_t prte_ras_gridengine_module = { .allocate = prte_ras_gridengine_allocate, .finalize = prte_ras_gridengine_finalize }; /** * Discover available (pre-allocated) nodes. Allocate the * requested number of nodes/process slots to the job. * */ static int prte_ras_gridengine_allocate(prte_job_t *jdata, pmix_list_t *nodelist) { char *pe_hostfile; char *job_id; char buf[1024], *tok, *num, *queue, *arch, *ptr; int rc; FILE *fp; prte_node_t *node; bool found; PRTE_HIDE_UNUSED_PARAMS(jdata); pe_hostfile = getenv("PE_HOSTFILE"); job_id = getenv("JOB_ID"); if (NULL == pe_hostfile || NULL == job_id) { return PRTE_ERR_TAKE_NEXT_OPTION; } /* show the Grid Engine's JOB_ID */ if (prte_mca_ras_gridengine_component.show_jobid || prte_mca_ras_gridengine_component.verbose != -1) { pmix_output(0, "ras:gridengine: JOB_ID: %s", job_id); } /* check the PE_HOSTFILE before continuing on */ fp = fopen(pe_hostfile, "r"); if (NULL == fp) { pmix_show_help("help-ras-gridengine.txt", "cannot-read-pe-hostfile", true, pe_hostfile, strerror(errno)); rc = PRTE_ERROR; PRTE_ERROR_LOG(rc); goto cleanup; } /* parse the pe_hostfile for hostname, slots, etc, then compare the * current node with a list of hosts in the nodelist, if the current * node is not found in nodelist, add it in */ pmix_output(prte_mca_ras_gridengine_component.verbose, "ras:gridengine: PE_HOSTFILE: %s", pe_hostfile); while (fgets(buf, sizeof(buf), fp)) { ptr = strtok_r(buf, " \n", &tok); num = strtok_r(NULL, " \n", &tok); queue = strtok_r(NULL, " \n", &tok); arch = strtok_r(NULL, " \n", &tok); /* see if we already have this node */ found = false; PMIX_LIST_FOREACH(node, nodelist, prte_node_t) { if (0 == strcmp(ptr, node->name)) { /* just add the slots */ node->slots += (int) strtol(num, (char **) NULL, 10); found = true; break; } } if (!found) { /* create a new node entry */ node = PMIX_NEW(prte_node_t); if (NULL == node) { fclose(fp); return PRTE_ERR_OUT_OF_RESOURCE; } node->name = strdup(ptr); node->state = PRTE_NODE_STATE_UP; node->slots_inuse = 0; node->slots_max = 0; node->slots = (int) strtol(num, (char **) NULL, 10); pmix_output(prte_mca_ras_gridengine_component.verbose, "ras:gridengine: %s: PE_HOSTFILE shows slots=%d queue=%s arch=%s", node->name, node->slots, queue, arch); pmix_list_append(nodelist, &node->super); } } /* finished reading the $PE_HOSTFILE */ cleanup: if (NULL != fp) { fclose(fp); } /* in gridengine, if we didn't find anything, then something * is wrong. The user may not have indicated this was a parallel * job, or may not have an allocation at all. In any case, this * is considered an unrecoverable error and we need to report it */ if (pmix_list_is_empty(nodelist)) { pmix_show_help("help-ras-gridengine.txt", "no-nodes-found", true); return PRTE_ERR_NOT_FOUND; } return PRTE_SUCCESS; } /** * finalize */ static int prte_ras_gridengine_finalize(void) { /* Nothing to do */ pmix_output(prte_mca_ras_gridengine_component.verbose, "ras:gridengine:finalize: success (nothing to do)"); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/gridengine/help-ras-gridengine.txt0000664000175000017500000000316015203542132024453 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # [cannot-read-pe-hostfile] The Grid Engine ras component is not able to read the $PE_HOSTFILE for the Grid Engine nodes. The $PE_HOSTFILE environment variable shows the file is located at: %s The following error is returned: %s # [no-nodes-found] No slots were found in the Grid Engine Parallel Environment (PE). This is probably a setup issue, or could be due to a corrupted $PE_HOSTFILE. There is also a chance that you may not be running under a parallel job environment (PE) -- you may have forgotten to specify this is a parallel job. If so, please specify that to Grid Engine and try again. prrte-4.1.1~rc2/src/mca/ras/gridengine/Makefile.am0000664000175000017500000000370515203542132022127 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = help-ras-gridengine.txt sources = \ ras_gridengine.h \ ras_gridengine_component.c \ ras_gridengine_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_gridengine_DSO component_noinst = component_install = prte_mca_ras_gridengine.la else component_noinst = libprtemca_ras_gridengine.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ras_gridengine_la_SOURCES = $(sources) prte_mca_ras_gridengine_la_LDFLAGS = -module -avoid-version prte_mca_ras_gridengine_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ras_gridengine_la_SOURCES =$(sources) libprtemca_ras_gridengine_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ras/testrm/0000775000175000017500000000000015203542132017271 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ras/testrm/ras_testrm.c0000664000175000017500000000243215203542132021621 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include "src/class/pmix_list.h" #include "src/runtime/prte_globals.h" #include "src/util/hostfile/hostfile.h" #include "ras_testrm.h" /* * Local functions */ static int allocate(prte_job_t *jdata, pmix_list_t *nodes); static int finalize(void); /* * Global variable */ prte_ras_base_module_t prte_ras_testrm_module = { .init = NULL, .allocate = allocate, .deallocate = NULL, .finalize = finalize }; static int allocate(prte_job_t *jdata, pmix_list_t *nodes) { int rc; PRTE_HIDE_UNUSED_PARAMS(jdata); rc = prte_util_add_hostfile_nodes(nodes, prte_mca_ras_testrm_component.hostfile); return rc; } /* * There's really nothing to do here */ static int finalize(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ras/testrm/Makefile.am0000664000175000017500000000222715203542132021330 0ustar alastairalastair# # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2023 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ras_testrm.h \ ras_testrm_component.c \ ras_testrm.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ras_testrm_DSO lib = lib_sources = component = prte_mca_ras_testrm.la component_sources = $(sources) else lib = libprtemca_ras_testrm.la lib_sources = $(sources) component = component_sources = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component) prte_mca_ras_testrm_la_SOURCES = $(component_sources) prte_mca_ras_testrm_la_LDFLAGS = -module -avoid-version prte_mca_ras_testrm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(lib) libprtemca_ras_testrm_la_SOURCES = $(lib_sources) libprtemca_ras_testrm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ras/testrm/ras_testrm.h0000664000175000017500000000162015203542132021624 0ustar alastairalastair/* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_RAS_TESTRM_H #define PRTE_RAS_TESTRM_H #include "prte_config.h" #include "src/mca/ras/base/base.h" #include "src/mca/ras/ras.h" BEGIN_C_DECLS struct prte_ras_testrm_component_t { prte_ras_base_component_t super; char *hostfile; }; typedef struct prte_ras_testrm_component_t prte_ras_testrm_component_t; PRTE_EXPORT extern prte_ras_testrm_component_t prte_mca_ras_testrm_component; PRTE_EXPORT extern prte_ras_base_module_t prte_ras_testrm_module; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ras/testrm/ras_testrm_component.c0000664000175000017500000000604615203542132023710 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "ras_testrm.h" #include "src/mca/ras/base/base.h" /* * Local functions */ static int ras_testrm_register(void); static int ras_testrm_component_query(pmix_mca_base_module_t **module, int *priority); prte_ras_testrm_component_t prte_mca_ras_testrm_component = { .super = { PRTE_RAS_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "testrm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_query_component = ras_testrm_component_query, .pmix_mca_register_component_params = ras_testrm_register } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ras, testrm) static int ras_testrm_register(void) { pmix_mca_base_component_t *component = &prte_mca_ras_testrm_component.super; prte_mca_ras_testrm_component.hostfile = NULL; (void) pmix_mca_base_component_var_register(component, "hostfile", "Name of file containing hosts for allocation", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_ras_testrm_component.hostfile); return PRTE_SUCCESS; } static int ras_testrm_component_query(pmix_mca_base_module_t **module, int *priority) { if (NULL != prte_mca_ras_testrm_component.hostfile) { *module = (pmix_mca_base_module_t *) &prte_ras_testrm_module; *priority = 1000; return PRTE_SUCCESS; } /* Sadly, no */ *module = NULL; *priority = 0; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/ess/0000775000175000017500000000000015203542132015760 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/hnp/0000775000175000017500000000000015203542132016545 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/hnp/owner.txt0000664000175000017500000000024415203542132020440 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ess/hnp/ess_hnp.h0000664000175000017500000000265715203542132020367 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ESS_HNP_H #define PRTE_ESS_HNP_H BEGIN_C_DECLS /* * Module open / close */ PRTE_MODULE_EXPORT extern prte_ess_base_component_t prte_mca_ess_hnp_component; END_C_DECLS #endif /* PRTE_ESS_HNP_H */ prrte-4.1.1~rc2/src/mca/ess/hnp/ess_hnp_component.c0000664000175000017500000000642015203542132022434 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "src/util/pmix_argv.h" #include "prte_config.h" #include "constants.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/hnp/ess_hnp.h" #include "src/runtime/prte_globals.h" extern prte_ess_base_module_t prte_ess_hnp_module; static int hnp_component_open(void); static int hnp_component_close(void); static int hnp_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_ess_base_component_t prte_mca_ess_hnp_component = { PRTE_ESS_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "hnp", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = hnp_component_open, .pmix_mca_close_component = hnp_component_close, .pmix_mca_query_component = hnp_component_query }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ess, hnp) static int hnp_component_open(void) { return PRTE_SUCCESS; } static int hnp_component_query(pmix_mca_base_module_t **module, int *priority) { /* we are the hnp module - we need to be selected * IFF we are designated as the hnp */ if (PRTE_PROC_IS_MASTER) { *priority = 100; *module = (pmix_mca_base_module_t *) &prte_ess_hnp_module; return PRTE_SUCCESS; } /* else, we are not */ *priority = -1; *module = NULL; return PRTE_ERROR; } static int hnp_component_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/hnp/ess_hnp_module.c0000664000175000017500000003776515203542132021737 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/include/hash_string.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_fd.h" #include "src/util/pmix_if.h" #include "src/util/malloc.h" #include "src/util/pmix_os_path.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/filem/base/base.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/base/base.h" #include "src/mca/plm/plm.h" #include "src/mca/prtereachable/base/base.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/schizo/base/base.h" #include "src/mca/state/base/base.h" #include "src/mca/state/state.h" #include "src/prted/pmix/pmix_server.h" #include "src/rml/rml.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/runtime/runtime.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/hnp/ess_hnp.h" static int rte_init(int argc, char **argv); static int rte_finalize(void); prte_ess_base_module_t prte_ess_hnp_module = { .init = rte_init, .finalize = rte_finalize }; static int rte_init(int argc, char **argv) { int ret; char *error = NULL; char *tmp; prte_job_t *jdata = NULL; prte_node_t *node; prte_proc_t *proc; prte_app_context_t *app; int idx; prte_topology_t *t; PRTE_HIDE_UNUSED_PARAMS(argc); /* run the prolog */ if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) { error = "prte_ess_base_std_prolog"; goto error; } /* get the local topology */ if (NULL == prte_hwloc_topology) { if (PRTE_SUCCESS != (ret = prte_hwloc_base_get_topology())) { error = "topology discovery"; goto error; } } /* open and setup the state machine */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_state_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { error = "prte_state_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_state_base_select())) { error = "prte_state_base_select"; goto error; } /* open the errmgr */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_errmgr_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { error = "prte_errmgr_base_open"; goto error; } /* Since we are the HNP, then responsibility for * defining the name falls to the PLM component for our * respective environment - hence, we have to open the PLM * first and select that component. */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_plm_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { error = "prte_plm_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_plm_base_select())) { error = "prte_plm_base_select"; if (PRTE_ERR_FATAL == ret) { /* we already output a show_help - so keep down the verbage */ ret = PRTE_ERR_SILENT; } goto error; } if (PRTE_SUCCESS != (ret = prte_plm.set_hnp_name())) { error = "prte_plm_set_hnp_name"; goto error; } /* get the job data object for the daemons */ jdata = PMIX_NEW(prte_job_t); PMIX_LOAD_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace); ret = prte_set_job_data_object(jdata); /* set the schizo personality to "prte" by default */ jdata->schizo = (struct prte_schizo_base_module_t*)prte_schizo_base_detect_proxy("prte"); if (NULL == jdata->schizo) { pmix_show_help("help-schizo-base.txt", "no-proxy", true, prte_tool_basename, "prte"); error = "select personality"; ret = PRTE_ERR_SILENT; goto error; } /* Set the session of the daemon job to the default session */ jdata->session = prte_default_session; /* mark that the daemons have reported as we are the * only ones in the system right now, and we definitely * are running! */ jdata->state = PRTE_JOB_STATE_DAEMONS_REPORTED; /* every job requires at least one app */ app = PMIX_NEW(prte_app_context_t); app->app = strdup(argv[0]); app->argv = PMIx_Argv_copy(argv); app->job = (struct prte_job_t*)jdata; pmix_pointer_array_set_item(jdata->apps, 0, app); jdata->num_apps++; /* create and store a node object where we are */ node = PMIX_NEW(prte_node_t); node->name = strdup(prte_process_info.nodename); node->index = PRTE_PROC_MY_NAME->rank; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_LOC_VERIFIED); pmix_pointer_array_set_item(prte_node_pool, PRTE_PROC_MY_NAME->rank, node); /* create and store a proc object for us */ proc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&proc->name, PRTE_PROC_MY_NAME->nspace, PRTE_PROC_MY_NAME->rank); proc->pid = prte_process_info.pid; proc->state = PRTE_PROC_STATE_RUNNING; PMIX_RETAIN(node); /* keep accounting straight */ proc->node = node; pmix_pointer_array_set_item(jdata->procs, PRTE_PROC_MY_NAME->rank, proc); /* record that the daemon (i.e., us) is on this node * NOTE: we do not add the proc object to the node's * proc array because we are not an application proc. * Instead, we record it in the daemon field of the * node object */ PMIX_RETAIN(proc); /* keep accounting straight */ node->daemon = proc; PRTE_FLAG_SET(node, PRTE_NODE_FLAG_DAEMON_LAUNCHED); node->state = PRTE_NODE_STATE_UP; /* record that the daemon job is running */ jdata->num_procs = 1; jdata->state = PRTE_JOB_STATE_RUNNING; /* obviously, we have "reported" */ jdata->num_reported = 1; jdata->num_daemons_reported = 1; /* setup my session directory here */ PMIX_OUTPUT_VERBOSE((2, prte_debug_output, "%s setting up session dir with\n\ttmpdir: %s\n\thost %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == prte_process_info.tmpdir_base) ? "UNDEF" : prte_process_info.tmpdir_base, prte_process_info.nodename)); /* create the directory tree */ if (PRTE_SUCCESS != (ret = prte_session_dir(PRTE_PROC_MY_NAME))) { error = "prte_session_dir"; goto error; } /* setup the PMIx server - we need this here in case the * communications infrastructure wants to register * information */ if (PRTE_SUCCESS != (ret = pmix_server_init())) { /* the server code already barked, so let's be quiet */ ret = PRTE_ERR_SILENT; error = "pmix_server_init"; goto error; } /* add network aliases to our list of alias hostnames - must * wait until after we init PMIx before getting them */ pmix_ifgetaliases(&prte_process_info.aliases); /* get our aliases - will include all the interface aliases captured in prte_init */ node->aliases = PMIx_Argv_copy(prte_process_info.aliases); /* if we are using xml for output, put a start tag */ if (prte_xml_output) { fprintf(stdout, "<%s>\n", prte_tool_basename); fflush(stdout); } /* Setup the communication infrastructure */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_prtereachable_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_prtereachable_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_reachable_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_prtereachable_base_select"; goto error; } if (PRTE_SUCCESS != (ret = prte_rml_open())) { PRTE_ERROR_LOG(ret); error = "prte_rml_open"; goto error; } /* it is now safe to start the pmix server */ pmix_server_start(); /* * Group communications */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_grpcomm_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_grpcomm_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_grpcomm_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_grpcomm_base_select"; goto error; } /* setup the error manager */ if (PRTE_SUCCESS != (ret = prte_errmgr_base_select())) { error = "prte_errmgr_base_select"; goto error; } if (0 < pmix_output_get_verbosity(prte_ess_base_framework.framework_output)) { pmix_output(0, "ALIASES FOR %s", node->name); if (NULL != node->aliases) { for (idx=0; NULL != node->aliases[idx]; idx++) { pmix_output(0, "\tALIAS: %s", node->aliases[idx]); } } } /* Now provide a chance for the PLM * to perform any module-specific init functions. This * needs to occur AFTER the communications are setup * as it may involve starting a non-blocking recv */ if (PRTE_SUCCESS != (ret = prte_plm.init())) { PRTE_ERROR_LOG(ret); error = "prte_plm_init"; goto error; } /* * Setup the remaining resource * management and errmgr frameworks - application procs * and daemons do not open these frameworks as they only use * the hnp proxy support in the PLM framework. */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_ras_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_ras_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_ras_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_ras_base_find_available"; goto error; } if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_rmaps_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_rmaps_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_rmaps_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_rmaps_base_find_available"; goto error; } /* add the topology to the array of known topologies */ t = PMIX_NEW(prte_topology_t); t->topo = prte_hwloc_topology; t->index = pmix_pointer_array_add(prte_node_topologies, t); node->topology = t; node->available = prte_hwloc_base_filter_cpus(prte_hwloc_topology); if (15 < pmix_output_get_verbosity(prte_ess_base_framework.framework_output)) { char *output = NULL; pmix_output(0, "%s Topology Info:", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); prte_hwloc_print(&output, "\t", prte_hwloc_topology); pmix_output(0, "%s", output); free(output); } /* Open/select the odls */ ret = pmix_mca_base_framework_open(&prte_odls_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); error = "prte_odls_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_odls_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_odls_base_select"; goto error; } /* set the pmix_output hnp file location to be in the * proc-specific session directory. */ pmix_asprintf(&tmp, "%s/%s", jdata->session_dir, PMIX_RANK_PRINT(PRTE_PROC_MY_NAME->rank)); pmix_output_set_output_file_info(tmp, "output-", NULL, NULL); free(tmp); /* setup I/O forwarding system - must come after we init routes */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_iof_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_iof_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_iof_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_iof_base_select"; goto error; } /* setup the FileM */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_filem_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_filem_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_filem_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_filem_base_select"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret && !prte_report_silent_errors) { pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } if (NULL != jdata) { /* remove our session directory tree */ PMIX_RELEASE(jdata); } return PRTE_ERR_SILENT; } static int rte_finalize(void) { /* first stage shutdown of the errmgr, deregister the handler but keep * the required facilities until the rml is offline */ prte_errmgr.finalize(); /* close frameworks */ (void) pmix_mca_base_framework_close(&prte_filem_base_framework); (void) pmix_mca_base_framework_close(&prte_grpcomm_base_framework); (void) pmix_mca_base_framework_close(&prte_iof_base_framework); (void) pmix_mca_base_framework_close(&prte_plm_base_framework); if (!prte_abnormal_term_ordered) { /* make sure our local procs are dead */ prte_odls.kill_local_procs(NULL); } (void) pmix_mca_base_framework_close(&prte_odls_base_framework); prte_rml_close(); (void) pmix_mca_base_framework_close(&prte_prtereachable_base_framework); (void) pmix_mca_base_framework_close(&prte_errmgr_base_framework); (void) pmix_mca_base_framework_close(&prte_state_base_framework); if (prte_xml_output) { fprintf(stdout, "\n", prte_tool_basename); fflush(stdout); } /* shutdown the pmix server */ pmix_server_finalize(); /* output any lingering stdout/err data */ fflush(stdout); fflush(stderr); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/hnp/Makefile.am0000664000175000017500000000347415203542132020611 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 Los Alamos National Security, LLC. All rights # reseved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ess_hnp.h \ ess_hnp_component.c \ ess_hnp_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ess_hnp_DSO component_noinst = component_install = prte_mca_ess_hnp.la else component_noinst = libprtemca_ess_hnp.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ess_hnp_la_SOURCES = $(sources) prte_mca_ess_hnp_la_LDFLAGS = -module -avoid-version prte_mca_ess_hnp_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ess_hnp_la_SOURCES =$(sources) libprtemca_ess_hnp_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ess/lsf/0000775000175000017500000000000015203542132016544 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/lsf/owner.txt0000664000175000017500000000024415203542132020437 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ess/lsf/ess_lsf.h0000664000175000017500000000265715203542132020365 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ESS_LSF_H #define PRTE_ESS_LSF_H BEGIN_C_DECLS PRTE_MODULE_EXPORT extern prte_ess_base_component_t prte_mca_ess_lsf_component; /* * Module open / close */ int prte_mca_ess_lsf_component_open(void); int prte_mca_ess_lsf_component_close(void); int prte_mca_ess_lsf_component_query(pmix_mca_base_module_t **module, int *priority); END_C_DECLS #endif /* PRTE_ESS_LSF_H */ prrte-4.1.1~rc2/src/mca/ess/lsf/configure.m40000664000175000017500000000277715203542132021004 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ess_lsf_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ess_lsf_CONFIG],[ AC_CONFIG_FILES([src/mca/ess/lsf/Makefile]) PRTE_CHECK_LSF([ess_lsf], [ess_lsf_good=1], [ess_lsf_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$ess_lsf_good" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([ess_lsf_CPPFLAGS]) AC_SUBST([ess_lsf_LDFLAGS]) AC_SUBST([ess_lsf_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/ess/lsf/Makefile.am0000664000175000017500000000350715203542132020605 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # The LSF plugin does not call any LSF library functions - it # therefore does NOT need to link against LSF libraries sources = \ ess_lsf.h \ ess_lsf_component.c \ ess_lsf_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ess_lsf_DSO component_noinst = component_install = prte_mca_ess_lsf.la else component_noinst = libprtemca_ess_lsf.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ess_lsf_la_SOURCES = $(sources) prte_mca_ess_lsf_la_LDFLAGS = -module -avoid-version prte_mca_ess_lsf_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ess_lsf_la_SOURCES =$(sources) libprtemca_ess_lsf_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ess/lsf/ess_lsf_module.c0000664000175000017500000000712515203542132021720 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2016-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include "src/util/pmix_argv.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/errmgr.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/lsf/ess_lsf.h" static int lsf_set_name(void); static int rte_init(int argc, char **argv); static int rte_finalize(void); prte_ess_base_module_t prte_ess_lsf_module = { .init = rte_init, .finalize = rte_finalize }; /* * Local variables */ static prte_node_rank_t my_node_rank = PRTE_NODE_RANK_INVALID; static int rte_init(int argc, char **argv) { int ret; char *error = NULL; /* run the prolog */ if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) { error = "prte_ess_base_std_prolog"; goto error; } /* Start by getting a unique name */ lsf_set_name(); if (PRTE_SUCCESS != (ret = prte_ess_base_prted_setup())) { PRTE_ERROR_LOG(ret); error = "prte_ess_base_prted_setup"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret && !prte_report_silent_errors) { pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } static int rte_finalize(void) { int ret; if (PRTE_SUCCESS != (ret = prte_ess_base_prted_finalize())) { PRTE_ERROR_LOG(ret); } return ret; ; } static int lsf_set_name(void) { int lsf_nodeid; pmix_rank_t vpid; if (NULL == prte_ess_base_nspace) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } PMIX_LOAD_NSPACE(PRTE_PROC_MY_NAME->nspace, prte_ess_base_nspace); if (NULL == prte_ess_base_vpid) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } vpid = strtoul(prte_ess_base_vpid, NULL, 10); lsf_nodeid = atoi(getenv("LSF_PM_TASKID")); pmix_output_verbose(1, prte_ess_base_framework.framework_output, "ess:lsf found LSF_PM_TASKID set to %d", lsf_nodeid); PRTE_PROC_MY_NAME->rank = vpid + lsf_nodeid - 1; /* get the num procs as provided in the cmd line param */ prte_process_info.num_daemons = prte_ess_base_num_procs; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/lsf/ess_lsf_component.c0000664000175000017500000000562315203542132022436 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/util/proc_info.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/lsf/ess_lsf.h" extern prte_ess_base_module_t prte_ess_lsf_module; /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_ess_base_component_t prte_mca_ess_lsf_component = { PRTE_ESS_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "lsf", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = prte_mca_ess_lsf_component_open, .pmix_mca_close_component = prte_mca_ess_lsf_component_close, .pmix_mca_query_component = prte_mca_ess_lsf_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ess, lsf) int prte_mca_ess_lsf_component_open(void) { return PRTE_SUCCESS; } int prte_mca_ess_lsf_component_query(pmix_mca_base_module_t **module, int *priority) { /* Are we running under an LSF job? Were * we given a path back to the HNP? If the * answer to both is "yes", then we were launched * by mpirun in an LSF world */ if (PRTE_PROC_IS_DAEMON && NULL != getenv("LSB_JOBID") && NULL != prte_process_info.my_hnp_uri) { *priority = 40; *module = (pmix_mca_base_module_t *) &prte_ess_lsf_module; return PRTE_SUCCESS; } /* nope, not here */ *priority = -1; *module = NULL; return PRTE_ERROR; } int prte_mca_ess_lsf_component_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/slurm/0000775000175000017500000000000015203542132017122 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/slurm/owner.txt0000664000175000017500000000024415203542132021015 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/ess/slurm/configure.m40000664000175000017500000000302515203542132021345 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ess_slurm_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ess_slurm_CONFIG],[ AC_CONFIG_FILES([src/mca/ess/slurm/Makefile]) PRTE_CHECK_SLURM([ess_slurm], [ess_slurm_good=1], [ess_slurm_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$ess_slurm_good" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([ess_slurm_CPPFLAGS]) AC_SUBST([ess_slurm_LDFLAGS]) AC_SUBST([ess_slurm_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/ess/slurm/Makefile.am0000664000175000017500000000334715203542132021165 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ess_slurm.h \ ess_slurm_component.c \ ess_slurm_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ess_slurm_DSO component_noinst = component_install = prte_mca_ess_slurm.la else component_noinst = libprtemca_ess_slurm.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ess_slurm_la_SOURCES = $(sources) prte_mca_ess_slurm_la_LDFLAGS = -module -avoid-version prte_mca_ess_slurm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ess_slurm_la_SOURCES =$(sources) libprtemca_ess_slurm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ess/slurm/ess_slurm.h0000664000175000017500000000267515203542132021321 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ESS_SLURM_H #define PRTE_ESS_SLURM_H BEGIN_C_DECLS PRTE_MODULE_EXPORT extern prte_ess_base_component_t prte_mca_ess_slurm_component; /* * Module open / close */ int prte_mca_ess_slurm_component_open(void); int prte_mca_ess_slurm_component_close(void); int prte_mca_ess_slurm_component_query(pmix_mca_base_module_t **module, int *priority); END_C_DECLS #endif /* PRTE_ESS_SLURM_H */ prrte-4.1.1~rc2/src/mca/ess/slurm/ess_slurm_module.c0000664000175000017500000001056415203542132022655 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include "src/class/pmix_pointer_array.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/slurm/ess_slurm.h" static int slurm_set_name(void); static int rte_init(int argc, char **argv); static int rte_finalize(void); prte_ess_base_module_t prte_ess_slurm_module = { .init = rte_init, .finalize = rte_finalize }; static int rte_init(int argc, char **argv) { int ret; char *error = NULL; PRTE_HIDE_UNUSED_PARAMS(argc, argv); /* run the prolog */ if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) { error = "prte_ess_base_std_prolog"; goto error; } /* Start by getting a unique name */ slurm_set_name(); if (PRTE_SUCCESS != (ret = prte_ess_base_prted_setup())) { PRTE_ERROR_LOG(ret); error = "prte_ess_base_prted_setup"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret && !prte_report_silent_errors) { pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } static int rte_finalize(void) { int ret; if (PRTE_SUCCESS != (ret = prte_ess_base_prted_finalize())) { PRTE_ERROR_LOG(ret); } return ret; } static int slurm_set_name(void) { int slurm_nodeid; pmix_rank_t vpid; char *tmp; PMIX_OUTPUT_VERBOSE((1, prte_ess_base_framework.framework_output, "ess:slurm setting name")); if (NULL == prte_ess_base_nspace) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } PMIX_LOAD_NSPACE(PRTE_PROC_MY_NAME->nspace, prte_ess_base_nspace); if (NULL == prte_ess_base_vpid) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } vpid = strtoul(prte_ess_base_vpid, NULL, 10); /* fix up the vpid and make it the "real" vpid */ slurm_nodeid = atoi(getenv("SLURM_NODEID")); PRTE_PROC_MY_NAME->rank = vpid + slurm_nodeid; PMIX_OUTPUT_VERBOSE((1, prte_ess_base_framework.framework_output, "ess:slurm set name to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* fix up the system info nodename to match exactly what slurm returned */ if (NULL != prte_process_info.nodename) { free(prte_process_info.nodename); } if (NULL == (tmp = getenv("SLURMD_NODENAME"))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } prte_process_info.nodename = strdup(tmp); PMIX_OUTPUT_VERBOSE( (1, prte_ess_base_framework.framework_output, "ess:slurm set nodename to %s", (NULL == prte_process_info.nodename) ? "NULL" : prte_process_info.nodename)); /* get the num procs as provided in the cmd line param */ prte_process_info.num_daemons = prte_ess_base_num_procs; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/slurm/ess_slurm_component.c0000664000175000017500000000643215203542132023371 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include "src/util/proc_info.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/slurm/ess_slurm.h" extern prte_ess_base_module_t prte_ess_slurm_module; /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_ess_base_component_t prte_mca_ess_slurm_component = { PRTE_ESS_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "slurm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = prte_mca_ess_slurm_component_open, .pmix_mca_close_component = prte_mca_ess_slurm_component_close, .pmix_mca_query_component = prte_mca_ess_slurm_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ess, slurm) int prte_mca_ess_slurm_component_open(void) { return PRTE_SUCCESS; } int prte_mca_ess_slurm_component_query(pmix_mca_base_module_t **module, int *priority) { /* Are we running under a SLURM job? Were * we given a path back to the HNP? If the * answer to both is "yes", then we were launched * by mpirun in a slurm world, so make ourselves available */ if (PRTE_PROC_IS_DAEMON && NULL != getenv("SLURM_JOBID") && NULL != prte_process_info.my_hnp_uri) { *priority = 50; *module = (pmix_mca_base_module_t *) &prte_ess_slurm_module; return PRTE_SUCCESS; } /* Sadly, no */ *priority = -1; *module = NULL; return PRTE_ERROR; } int prte_mca_ess_slurm_component_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/env/0000775000175000017500000000000015203542132016550 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/env/ess_env.h0000664000175000017500000000265715203542132020375 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ESS_ENV_H #define PRTE_ESS_ENV_H BEGIN_C_DECLS /* * Module open / close */ int prte_mca_ess_env_component_open(void); int prte_mca_ess_env_component_close(void); int prte_mca_ess_env_component_query(pmix_mca_base_module_t **module, int *priority); PRTE_MODULE_EXPORT extern prte_ess_base_component_t prte_mca_ess_env_component; END_C_DECLS #endif /* PRTE_ESS_ENV_H */ prrte-4.1.1~rc2/src/mca/ess/env/ess_env_module.c0000664000175000017500000001011715203542132021723 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include #include "src/event/event-internal.h" #include "src/pmix/pmix-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_argv.h" #include "src/util/malloc.h" #include "src/util/pmix_output.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/iof/base/base.h" #include "src/mca/plm/base/base.h" #include "src/mca/ras/base/base.h" #include "src/rml/rml.h" #include "src/mca/filem/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/runtime/runtime.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/env/ess_env.h" #include "src/mca/ess/ess.h" static int env_set_name(void); static int rte_init(int argc, char **argv); static int rte_finalize(void); prte_ess_base_module_t prte_ess_env_module = { .init = rte_init, .finalize = rte_finalize }; static int rte_init(int argc, char **argv) { int ret; char *error = NULL; PRTE_HIDE_UNUSED_PARAMS(argc, argv); /* run the prolog */ if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) { error = "prte_ess_base_std_prolog"; goto error; } /* Start by getting a unique name from the enviro */ env_set_name(); /* if I am a daemon, complete my setup using the * default procedure */ if (PRTE_SUCCESS != (ret = prte_ess_base_prted_setup())) { PRTE_ERROR_LOG(ret); error = "prte_ess_base_prted_setup"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret && !prte_report_silent_errors) { pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } static int rte_finalize(void) { int ret; if (PRTE_SUCCESS != (ret = prte_ess_base_prted_finalize())) { PRTE_ERROR_LOG(ret); } return ret; } static int env_set_name(void) { pmix_rank_t vpid; if (NULL == prte_ess_base_nspace) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } PMIX_LOAD_NSPACE(PRTE_PROC_MY_NAME->nspace, prte_ess_base_nspace); if (NULL == prte_ess_base_vpid) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } vpid = strtoul(prte_ess_base_vpid, NULL, 10); PRTE_PROC_MY_NAME->rank = vpid; PMIX_OUTPUT_VERBOSE((1, prte_ess_base_framework.framework_output, "ess:env set name to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); prte_process_info.num_daemons = prte_ess_base_num_procs; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/env/ess_env_component.c0000664000175000017500000000621215203542132022441 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include "src/util/proc_info.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/env/ess_env.h" extern prte_ess_base_module_t prte_ess_env_module; /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_ess_base_component_t prte_mca_ess_env_component = { PRTE_ESS_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "env", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = prte_mca_ess_env_component_open, .pmix_mca_close_component = prte_mca_ess_env_component_close, .pmix_mca_query_component = prte_mca_ess_env_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ess, env) int prte_mca_ess_env_component_open(void) { return PRTE_SUCCESS; } int prte_mca_ess_env_component_query(pmix_mca_base_module_t **module, int *priority) { /* we are the env module, only used by daemons that are * launched by ssh so allow any enviro-specifc modules * to override us */ if (PRTE_PROC_IS_DAEMON) { *priority = 1; *module = (pmix_mca_base_module_t *) &prte_ess_env_module; return PRTE_SUCCESS; } /* if not, then return NULL - we cannot be selected */ *priority = -1; *module = NULL; return PRTE_ERROR; } int prte_mca_ess_env_component_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/env/owner.txt0000664000175000017500000000024415203542132020443 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: Intel status: maintenance prrte-4.1.1~rc2/src/mca/ess/env/Makefile.am0000664000175000017500000000332115203542132020603 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ess_env.h \ ess_env_component.c \ ess_env_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ess_env_DSO component_noinst = component_install = prte_mca_ess_env.la else component_noinst = libprtemca_ess_env.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ess_env_la_SOURCES = $(sources) prte_mca_ess_env_la_LDFLAGS = -module -avoid-version prte_mca_ess_env_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ess_env_la_SOURCES =$(sources) libprtemca_ess_env_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ess/base/0000775000175000017500000000000015203542132016672 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/base/owner.txt0000664000175000017500000000024615203542132020567 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: project status: maintenance prrte-4.1.1~rc2/src/mca/ess/base/ess_base_select.c0000664000175000017500000000373715203542132022173 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_component_repository.h" #include "src/mca/mca.h" #include "src/mca/ess/base/base.h" int prte_ess_base_select(void) { prte_ess_base_component_t *best_component = NULL; prte_ess_base_module_t *best_module = NULL; /* * Select the best component */ if (PRTE_SUCCESS != pmix_mca_base_select("ess", prte_ess_base_framework.framework_output, &prte_ess_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL)) { /* error message emitted by fn above */ return PRTE_ERR_SILENT; } /* Save the winner */ /* No global component structure */ prte_ess = *best_module; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/base/help-ess-base.txt0000664000175000017500000000255015203542132022065 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2024-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # This is the US/English general help file for the SDS base. # [ess-base:cannot-forward] The system does not support trapping and forwarding of the specified signal: signal: %s param: %s Please remove that signal from the ess_base_forward_signals MCA parameter. # [ess-base:unknown-signal] The following signal was included in the ess_base_forward_signals MCA parameter: signal: %s param: %s This is not a recognized signal value. Please fix or remove it. prrte-4.1.1~rc2/src/mca/ess/base/ess_base_bootstrap.c0000664000175000017500000004300715203542132022723 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2023 Triad National Security, LLC. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include "src/util/proc_info.h" #include "src/event/event-internal.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_os_path.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_string_copy.h" #include "src/mca/prteinstalldirs/base/base.h" #include "src/rml/rml.h" #include "src/rml/rml_contact.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" static pmix_status_t regex_extract_nodes(char *regexp, char ***names); static pmix_status_t regex_parse_value_ranges(char *base, char *ranges, int num_digits, char *suffix, char ***names); static pmix_status_t regex_parse_value_range(char *base, char *range, int num_digits, char *suffix, char ***names); static pmix_status_t read_file(char *regexp, char ***names); int prte_ess_base_bootstrap(void) { char *path, *line, *ptr; FILE *fp; int n; char *cluster = NULL; char *ctrlhost = NULL; uint32_t ctrlport = UINT32_MAX; uint32_t prtedport = UINT32_MAX; char *dvmnodes = NULL; char *dvmtmpdir = NULL; char *sessiontmpdir = NULL; char *ctrllogpath = NULL; char *prtedlogpath = NULL; char **nodes = NULL; int rc = PRTE_ERR_SILENT; /* see if we can open a configuration file */ path = pmix_os_path(false, prte_install_dirs.sysconfdir, "prte.conf", NULL); fp = fopen(path, "r"); if (NULL == fp) { pmix_show_help("help-prte-runtime.txt", "bootstrap-not-found", true, prte_process_info.nodename, path); free(path); return PRTE_ERR_SILENT; } while (NULL != (line = pmix_getline(fp))) { /* ignore if line is empty or comment */ if (0 == strlen(line) || '#' == line[0]) { free(line); continue; } /* split on the '=' sign */ if (NULL == (ptr = strchr(line, '='))) { /* bad file */ pmix_show_help("help-prte-runtime.txt", "bootstrap-bad-entry", true, prte_process_info.nodename, path, line); free(path); fclose(fp); rc = PRTE_ERR_SILENT; goto cleanup; } *ptr = '\0'; if (0 == strlen(line)) { // missing the field name /* restore the '=' sign */ *ptr = '='; pmix_show_help("help-prte-runtime.txt", "bootstrap-missing-field-name", true, prte_process_info.nodename, path, ptr); free(path); fclose(fp); rc = PRTE_ERR_SILENT; goto cleanup; } ++ptr; if ('\0' == *ptr) { // missing the value pmix_show_help("help-prte-runtime.txt", "bootstrap-missing-value", true, prte_process_info.nodename, path, line); free(path); fclose(fp); rc = PRTE_ERR_SILENT; goto cleanup; } /* identify and cache the option */ if (0 == strcmp(line, "ClusterName")) { if (NULL != cluster) { free(cluster); } cluster = strdup(ptr); } else if (0 == strcmp(line, "DVMControllerHost")) { if (NULL != ctrlhost) { free(ctrlhost); } ctrlhost = strdup(ptr); } else if (0 == strcmp(line, "DVMControllerPort")) { ctrlport = strtoul(ptr, NULL, 10); } else if (0 == strcmp(line, "PRTEDPort")) { prtedport = strtoul(ptr, NULL, 10); } else if (0 == strcmp(line, "DVMNodes")) { if (NULL != dvmnodes) { free(dvmnodes); } dvmnodes = strdup(ptr); } else if (0 == strcmp(line, "DVMTempDir")) { if (NULL != dvmtmpdir) { free(dvmtmpdir); } dvmtmpdir = strdup(ptr); } else if (0 == strcmp(line, "SessionTmpDir")) { if (NULL != sessiontmpdir) { free(sessiontmpdir); } sessiontmpdir = strdup(ptr); } else if (0 == strcmp(line, "ControllerLogPath")) { if (NULL != ctrllogpath) { free(ctrllogpath); } ctrllogpath = strdup(ptr); } else if (0 == strcmp(line, "PRTEDLogPath")) { if (NULL != prtedlogpath) { free(prtedlogpath); } prtedlogpath = strdup(ptr); } free(line); } fclose(fp); /* we require the node list */ if (NULL == dvmnodes) { pmix_show_help("help-prte-runtime.txt", "bootstrap-missing-entry", true, prte_process_info.nodename, path, "DVMNodes"); goto cleanup; } /* we must be able to parse that list */ rc = regex_extract_nodes(dvmnodes, &nodes); if (PMIX_SUCCESS != rc) { pmix_show_help("help-prte-runtime.txt", "bootstrap-bad-nodelist", true, prte_process_info.nodename, path, dvmnodes, PMIx_Error_string(rc)); goto cleanup; } /* we must have the controller host so we can find it */ if (NULL == ctrlhost) { pmix_show_help("help-prte-runtime.txt", "bootstrap-missing-entry", true, prte_process_info.nodename, path, "DVMControllerHost"); goto cleanup; } /* we must have a prted port so we can contact our parent */ if (UINT32_MAX == prtedport) { pmix_show_help("help-prte-runtime.txt", "bootstrap-missing-entry", true, prte_process_info.nodename, path, "DVMControllerPort"); goto cleanup; } /* if we aren't given a controller port, default to the prted port */ if (UINT32_MAX == ctrlport) { ctrlport = prtedport; } /* report the nodes */ for (n=0; NULL != nodes[n]; n++) { pmix_output(0, "NODE[%d]: %s", n, nodes[n]); } rc = PRTE_SUCCESS; cleanup: if (NULL != cluster) { free(cluster); } if (NULL != ctrlhost) { free(ctrlhost); } if (NULL != dvmnodes) { free(dvmnodes); } if (NULL != nodes) { PMIx_Argv_free(nodes); } if (NULL != dvmtmpdir) { free(dvmtmpdir); } if (NULL != sessiontmpdir) { free(sessiontmpdir); } if (NULL != ctrllogpath) { free(ctrllogpath); } if (NULL != prtedlogpath) { free(prtedlogpath); } return rc; } static pmix_status_t regex_extract_nodes(char *regexp, char ***names) { int i, j, k, len; pmix_status_t ret; char *base; char *orig, *suffix; bool found_range = false; bool more_to_come = false; int num_digits; /* set the default */ *names = NULL; if (NULL == regexp) { return PMIX_ERR_BAD_PARAM; } /* see what regex we were given. Supported options: * * file: - file of names, one per line * - PMIx native regex, which is a comma-delimited * list of ranges or names */ if (0 == strncasecmp(regexp, "file:", 5)) { /* skip over the "file:" portion */ ret = read_file(®exp[5], names); return ret; } orig = base = strdup(regexp); if (NULL == base) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); return PMIX_ERR_OUT_OF_RESOURCE; } pmix_output_verbose(1, prte_ess_base_framework.framework_output, "bootstrap:extract:nodes: checking list: %s", regexp); do { /* Find the base */ len = strlen(base); for (i = 0; i <= len; ++i) { if (base[i] == '[') { /* we found a range. this gets dealt with below */ base[i] = '\0'; found_range = true; break; } if (base[i] == ',') { /* we found a singleton value, and there are more to come */ base[i] = '\0'; found_range = false; more_to_come = true; break; } if (base[i] == '\0') { /* we found a singleton value */ found_range = false; more_to_come = false; break; } } if (i == 0 && !found_range) { /* we found a special character at the beginning of the string */ free(orig); return PMIX_ERR_BAD_PARAM; } if (found_range) { /* If we found a range, get the number of digits in the numbers */ i++; /* step over the [ */ for (j = i; j < len; j++) { if (base[j] == ':') { base[j] = '\0'; break; } } if (j >= len) { /* we didn't find the number of digits */ free(orig); return PMIX_ERR_BAD_PARAM; } num_digits = strtol(&base[i], NULL, 10); i = j + 1; /* step over the : */ /* now find the end of the range */ for (j = i; j < len; ++j) { if (base[j] == ']') { base[j] = '\0'; break; } } if (j >= len) { /* we didn't find the end of the range */ free(orig); return PMIX_ERR_BAD_PARAM; } /* check for a suffix */ if (j + 1 < len && base[j + 1] != ',') { /* find the next comma, if present */ for (k = j + 1; k < len && base[k] != ','; k++) ; if (k < len) { base[k] = '\0'; } suffix = strdup(&base[j + 1]); if (k < len) { base[k] = ','; } j = k - 1; } else { suffix = NULL; } pmix_output_verbose(1, prte_ess_base_framework.framework_output, "bootstrap:extract:nodes: parsing range %s %s %s", base, base + i, suffix); ret = regex_parse_value_ranges(base, base + i, num_digits, suffix, names); if (NULL != suffix) { free(suffix); } if (PMIX_SUCCESS != ret) { free(orig); return ret; } if (j + 1 < len && base[j + 1] == ',') { more_to_come = true; base = &base[j + 2]; } else { more_to_come = false; } } else { /* If we didn't find a range, just add the value */ PMIx_Argv_append_nosize(names, base); /* step over the comma */ i++; /* set base equal to the (possible) next base to look at */ base = &base[i]; } } while (more_to_come); free(orig); /* All done */ return PRTE_SUCCESS; } /* * Parse one or more ranges in a set * * @param base The base text of the value name * @param *ranges A pointer to a range. This can contain multiple ranges * (i.e. "1-3,10" or "5" or "9,0100-0130,250") * @param ***names An argv array to add the newly discovered values to */ static pmix_status_t regex_parse_value_ranges(char *base, char *ranges, int num_digits, char *suffix, char ***names) { int i, len; pmix_status_t ret; char *start, *orig; /* Look for commas, the separator between ranges */ len = strlen(ranges); for (orig = start = ranges, i = 0; i < len; ++i) { if (',' == ranges[i]) { ranges[i] = '\0'; ret = regex_parse_value_range(base, start, num_digits, suffix, names); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); return ret; } start = ranges + i + 1; } } /* Pick up the last range, if it exists */ if (start < orig + len) { pmix_output_verbose(1, prte_ess_base_framework.framework_output, "bootstrap:parse:ranges: parse range %s (2)", start); ret = regex_parse_value_range(base, start, num_digits, suffix, names); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); return ret; } } /* All done */ return PMIX_SUCCESS; } /* * Parse a single range in a set and add the full names of the values * found to the names argv * * @param base The base text of the value name * @param *ranges A pointer to a single range. (i.e. "1-3" or "5") * @param ***names An argv array to add the newly discovered values to */ static pmix_status_t regex_parse_value_range(char *base, char *range, int num_digits, char *suffix, char ***names) { char *str, tmp[132]; size_t i, k, start, end; size_t base_len, len; bool found; if (NULL == base || NULL == range) { return PMIX_ERROR; } len = strlen(range); base_len = strlen(base); /* Silence compiler warnings; start and end are always assigned properly, below */ start = end = 0; /* Look for the beginning of the first number */ for (found = false, i = 0; i < len; ++i) { if (isdigit((int) range[i])) { if (!found) { start = strtol(range + i, NULL, 10); found = true; break; } } } if (!found) { PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); return PMIX_ERR_NOT_FOUND; } /* Look for the end of the first number */ for (found = false; i < len; ++i) { if (!isdigit(range[i])) { break; } } /* Was there no range, just a single number? */ if (i >= len) { end = start; found = true; } else { /* Nope, there was a range. Look for the beginning of the second * number */ for (; i < len; ++i) { if (isdigit(range[i])) { end = strtol(range + i, NULL, 10); found = true; break; } } } if (!found) { PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); return PMIX_ERR_NOT_FOUND; } /* Make strings for all values in the range */ len = base_len + num_digits + 32; if (NULL != suffix) { len += strlen(suffix); } str = (char *) malloc(len); if (NULL == str) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); return PMIX_ERR_OUT_OF_RESOURCE; } for (i = start; i <= end; ++i) { memset(str, 0, len); strcpy(str, base); /* we need to zero-pad the digits */ for (k = 0; k < (size_t) num_digits; k++) { str[k + base_len] = '0'; } memset(tmp, 0, 132); pmix_snprintf(tmp, 132, "%lu", (unsigned long) i); for (k = 0; k < strlen(tmp); k++) { str[base_len + num_digits - k - 1] = tmp[strlen(tmp) - k - 1]; } /* if there is a suffix, add it */ if (NULL != suffix) { strcat(str, suffix); } PMIx_Argv_append_nosize(names, str); } free(str); /* All done */ return PMIX_SUCCESS; } static pmix_status_t read_file(char *regexp, char ***names) { char *line; FILE *fp; fp = fopen(regexp, "r"); if (NULL == fp) { return PMIX_ERR_BAD_PARAM; } while (NULL != (line = pmix_getline(fp))) { /* ignore if line is empty or comment */ if (0 == strlen(line) || '#' == line[0]) { free(line); continue; } PMIx_Argv_append_nosize(names, line); free(line); } fclose(fp); return PMIX_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/base/ess_base_frame.c0000664000175000017500000002537615203542132022011 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" /* * The following file was created by configure. It contains extern * statements and the definition of an array of pointers to each * module's public pmix_mca_base_module_t struct. */ #include "src/mca/ess/base/static-components.h" prte_ess_base_module_t prte_ess = { .init = NULL, .finalize = NULL, }; int prte_ess_base_num_procs = -1; char *prte_ess_base_nspace = NULL; char *prte_ess_base_vpid = NULL; pmix_list_t prte_ess_base_signals = PMIX_LIST_STATIC_INIT; static char *forwarded_signals = "all"; static int prte_ess_base_register(pmix_mca_base_register_flag_t flags) { int ret; PRTE_HIDE_UNUSED_PARAMS(flags); prte_ess_base_nspace = NULL; ret = pmix_mca_base_var_register("prte", "ess", "base", "nspace", "Process nspace", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_ess_base_nspace); pmix_mca_base_var_register_synonym(ret, "prte", "prte", "ess", "nspace", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); prte_ess_base_vpid = NULL; ret = pmix_mca_base_var_register("prte", "ess", "base", "vpid", "Process vpid", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_ess_base_vpid); pmix_mca_base_var_register_synonym(ret, "prte", "prte", "ess", "vpid", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); prte_ess_base_num_procs = -1; ret = pmix_mca_base_var_register("prte", "ess", "base", "num_procs", "Used to discover the number of procs in the job", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_ess_base_num_procs); pmix_mca_base_var_register_synonym(ret, "prte", "prte", "ess", "num_procs", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); forwarded_signals = "all"; ret = pmix_mca_base_var_register("prte", "ess", "base", "forward_signals", "Comma-delimited list of signals (names or integers) to be forwarded to " "application processes [\"none\" => forward nothing, \"all\" => forward all]. " "Signals provided by default depends upon system definitions. The SIGTERM, SIGHUP, " "SIGINT signals are always forwarded regardless of this param's settings. The " "SIGKILL and SIGPIPE signals cannot be forwarded.", PMIX_MCA_BASE_VAR_TYPE_STRING, &forwarded_signals); pmix_mca_base_var_register_synonym(ret, "prte", "ess", "hnp", "forward_signals", PMIX_MCA_BASE_VAR_SYN_FLAG_DEPRECATED); return PRTE_SUCCESS; } static int prte_ess_base_close(void) { PMIX_LIST_DESTRUCT(&prte_ess_base_signals); return pmix_mca_base_framework_components_close(&prte_ess_base_framework, NULL); } static int prte_ess_base_open(pmix_mca_base_open_flag_t flags) { PMIX_CONSTRUCT(&prte_ess_base_signals, pmix_list_t); return pmix_mca_base_framework_components_open(&prte_ess_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, ess, "PRTE Environmenal System Setup", prte_ess_base_register, prte_ess_base_open, prte_ess_base_close, prte_ess_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); /* signal forwarding */ /* setup signal forwarding list */ struct known_signal { /** signal number */ int signal; /** signal name */ char *signame; /** can this signal be forwarded */ bool can_forward; }; static struct known_signal known_signals[] = { {SIGTERM, "SIGTERM", false}, {SIGHUP, "SIGHUP", false}, {SIGINT, "SIGINT", false}, {SIGKILL, "SIGKILL", false}, {SIGPIPE, "SIGPIPE", false}, #ifdef SIGQUIT {SIGQUIT, "SIGQUIT", false}, #endif #ifdef SIGTRAP {SIGTRAP, "SIGTRAP", true}, #endif #ifdef SIGTSTP {SIGTSTP, "SIGTSTP", true}, #endif #ifdef SIGABRT {SIGABRT, "SIGABRT", true}, #endif #ifdef SIGCONT {SIGCONT, "SIGCONT", true}, #endif #ifdef SIGSYS {SIGSYS, "SIGSYS", true}, #endif #ifdef SIGXCPU {SIGXCPU, "SIGXCPU", true}, #endif #ifdef SIGXFSZ {SIGXFSZ, "SIGXFSZ", true}, #endif #ifdef SIGALRM {SIGALRM, "SIGALRM", true}, #endif #ifdef SIGVTALRM {SIGVTALRM, "SIGVTALRM", true}, #endif #ifdef SIGPROF {SIGPROF, "SIGPROF", true}, #endif #ifdef SIGINFO {SIGINFO, "SIGINFO", true}, #endif #ifdef SIGPWR {SIGPWR, "SIGPWR", true}, #endif #ifdef SIGURG {SIGURG, "SIGURG", true}, #endif #ifdef SIGUSR1 {SIGUSR1, "SIGUSR1", true}, #endif #ifdef SIGUSR2 {SIGUSR2, "SIGUSR2", true}, #endif {0, NULL, false}, }; #define ESS_ADDSIGNAL(x, s) \ do { \ prte_ess_base_signal_t *_sig; \ _sig = PMIX_NEW(prte_ess_base_signal_t); \ _sig->signal = (x); \ _sig->signame = strdup((s)); \ pmix_list_append(&prte_ess_base_signals, &_sig->super); \ } while (0) static bool signals_added = false; pmix_status_t prte_ess_base_setup_signals(char *input) { int i, sval, nsigs; char *mysignals, **signals, *tmp, *sname=NULL; prte_ess_base_signal_t *sig; bool ignore, found; if (NULL == input) { mysignals = forwarded_signals; } else { mysignals = input; } /* if they told us "none", then nothing to do */ if (0 == strcasecmp(mysignals, "none") || signals_added) { return PMIX_SUCCESS; } signals_added = true; // only do this once // handle the "all" special case if (0 == strcasecmp(mysignals, "all")) { nsigs = sizeof(known_signals) / sizeof(struct known_signal); for (i = 0; i < nsigs; i++) { if (known_signals[i].can_forward) { pmix_output_verbose(2, prte_ess_base_framework.framework_output, "Forwarding signal: %s", known_signals[i].signame); ESS_ADDSIGNAL(known_signals[i].signal, known_signals[i].signame); } } return PMIX_SUCCESS; } /* see what they asked for - ignore any duplicates */ signals = PMIx_Argv_split(mysignals, ','); for (i = 0; NULL != signals[i]; i++) { sval = 0; if (0 != strncasecmp(signals[i], "SIG", 3)) { /* treat it like a number */ errno = 0; sval = strtoul(signals[i], &tmp, 10); if (0 != errno || '\0' != *tmp) { pmix_show_help("help-ess-base.txt", "ess-base:unknown-signal", true, signals[i], mysignals); PMIx_Argv_free(signals); return PMIX_ERR_SILENT; } // see if it's a known signal number sname = NULL; for (int j = 0; NULL != known_signals[j].signame; ++j) { if (sval == known_signals[j].signal) { sname = known_signals[j].signame; break; } } if (NULL == sname) { sname = signals[i]; } } else { /* they gave us a signal name */ found = false; for (int j = 0; NULL != known_signals[j].signame; ++j) { if (0 == strcasecmp(signals[i], known_signals[j].signame) || sval == known_signals[j].signal) { if (!known_signals[j].can_forward) { pmix_show_help("help-ess-base.txt", "ess-base:cannot-forward", true, known_signals[j].signame, mysignals); PMIx_Argv_free(signals); return PMIX_ERR_SILENT; } found = true; sval = known_signals[j].signal; sname = known_signals[j].signame; break; } } if (!found) { pmix_show_help("help-ess-base.txt", "ess-base:unknown-signal", true, signals[i], mysignals); PMIx_Argv_free(signals); return PMIX_ERR_SILENT; } } /* see if it is one we already covered */ ignore = false; PMIX_LIST_FOREACH(sig, &prte_ess_base_signals, prte_ess_base_signal_t) { if (0 == strcasecmp(sname, sig->signame) || sval == sig->signal) { /* got it - we will ignore */ ignore = true; break; } } if (ignore) { continue; } pmix_output_verbose(2, prte_ess_base_framework.framework_output, "Forwarding signal: %s", sname); ESS_ADDSIGNAL(sval, sname); } PMIx_Argv_free(signals); return PMIX_SUCCESS; } /* instantiate the class */ static void scon(prte_ess_base_signal_t *t) { t->signame = NULL; } static void sdes(prte_ess_base_signal_t *t) { if (NULL != t->signame) { free(t->signame); } } PMIX_CLASS_INSTANCE(prte_ess_base_signal_t, pmix_list_item_t, scon, sdes); prrte-4.1.1~rc2/src/mca/ess/base/base.h0000664000175000017500000000540415203542132017760 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef MCA_ESS_BASE_H #define MCA_ESS_BASE_H #include "prte_config.h" #include "types.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/mca/ess/ess.h" BEGIN_C_DECLS /* * MCA Framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_ess_base_framework; /** * Select a ess module */ PRTE_EXPORT int prte_ess_base_select(void); /* * stdout/stderr buffering control parameter */ PRTE_EXPORT extern int prte_ess_base_std_buffering; PRTE_EXPORT extern int prte_ess_base_num_procs; PRTE_EXPORT extern char *prte_ess_base_nspace; PRTE_EXPORT extern char *prte_ess_base_vpid; PRTE_EXPORT extern pmix_list_t prte_ess_base_signals; /* * Internal helper functions used by components */ PRTE_EXPORT int prte_ess_env_get(void); PRTE_EXPORT int prte_ess_base_std_prolog(void); PRTE_EXPORT int prte_ess_base_prted_setup(void); PRTE_EXPORT int prte_ess_base_prted_finalize(void); PRTE_EXPORT pmix_status_t prte_ess_base_setup_signals(char *signals); /* Detect whether or not this proc is bound - if not, * see if it should bind itself */ PRTE_EXPORT int prte_ess_base_proc_binding(void); /* * Put functions */ PRTE_EXPORT int prte_ess_env_put(int32_t num_procs, int32_t num_local_procs, char ***env); /* read a bootstrap configuration file */ PRTE_EXPORT int prte_ess_base_bootstrap(void); typedef struct { pmix_list_item_t super; char *signame; int signal; bool can_forward; } prte_ess_base_signal_t; PMIX_CLASS_DECLARATION(prte_ess_base_signal_t); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ess/base/Makefile.am0000664000175000017500000000257515203542132020737 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. # Copyright (c) 2015-2020 Intel, Inc. All rights reserved. # Copyright (c) 2019 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ base/help-ess-base.txt headers += \ base/base.h libprtemca_ess_la_SOURCES += \ base/ess_base_frame.c \ base/ess_base_select.c \ base/ess_base_std_prted.c \ base/ess_base_std_prolog.c \ base/ess_base_bootstrap.c prrte-4.1.1~rc2/src/mca/ess/base/ess_base_std_prted.c0000664000175000017500000004474515203542132022710 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2021 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include "src/util/proc_info.h" #include "src/event/event-internal.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_if.h" #include "src/util/pmix_os_path.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/filem/base/base.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/base/base.h" #include "src/mca/prtereachable/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/schizo/base/base.h" #include "src/mca/state/base/base.h" #include "src/mca/state/state.h" #include "src/prted/pmix/pmix_server.h" #include "src/rml/rml.h" #include "src/rml/rml_contact.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/util/name_fns.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" /* local globals */ static bool plm_in_use = false; static bool signals_set = false; static prte_event_t term_handler; static prte_event_t int_handler; static prte_event_t epipe_handler; static char *log_path = NULL; static void shutdown_signal(int fd, short flags, void *arg); static void epipe_signal_callback(int fd, short flags, void *arg); static void signal_forward_callback(int fd, short event, void *arg); static prte_event_t *forward_signals_events = NULL; static void setup_sighandler(int signal, prte_event_t *ev, prte_event_cbfunc_t cbfunc) { prte_event_signal_set(prte_event_base, ev, signal, cbfunc, ev); prte_event_signal_add(ev, NULL); } int prte_ess_base_prted_setup(void) { int ret = PRTE_ERROR; int fd = -1; char log_file[PRTE_PATH_MAX]; char *error = NULL; char *tmp = NULL; prte_job_t *jdata = NULL; prte_proc_t *proc; prte_app_context_t *app = NULL; prte_topology_t *t = NULL; prte_ess_base_signal_t *sig = NULL; int idx; plm_in_use = false; /* setup callback for SIGPIPE */ setup_sighandler(SIGPIPE, &epipe_handler, epipe_signal_callback); /* Set signal handlers to catch kill signals so we can properly clean up * after ourselves. */ setup_sighandler(SIGTERM, &term_handler, shutdown_signal); setup_sighandler(SIGINT, &int_handler, shutdown_signal); /** setup callbacks for signals we should forward */ if (0 < (idx = pmix_list_get_size(&prte_ess_base_signals))) { forward_signals_events = (prte_event_t *) malloc(sizeof(prte_event_t) * idx); if (NULL == forward_signals_events) { ret = PRTE_ERR_OUT_OF_RESOURCE; error = "unable to malloc"; goto error; } idx = 0; PMIX_LIST_FOREACH(sig, &prte_ess_base_signals, prte_ess_base_signal_t) { setup_sighandler(sig->signal, forward_signals_events + idx, signal_forward_callback); ++idx; } } signals_set = true; /* get the local topology */ if (NULL == prte_hwloc_topology) { if (PRTE_SUCCESS != (ret = prte_hwloc_base_get_topology())) { error = "topology discovery"; goto error; } } /* define the HNP name */ PMIX_LOAD_PROCID(PRTE_PROC_MY_HNP, PRTE_PROC_MY_NAME->nspace, 0); /* open and setup the state machine */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_state_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_state_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_state_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_state_base_select"; goto error; } /* open the errmgr */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_errmgr_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_errmgr_base_open"; goto error; } /* some environments allow remote launches - e.g., ssh - so * open and select something -only- if we are given * a specific module to use */ if (NULL != getenv("PRTE_MCA_plm")) { plm_in_use = true; if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_plm_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_plm_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_plm_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_plm_base_select"; goto error; } } /* Setup the job data object for the daemons */ /* create and store the job data object */ jdata = PMIX_NEW(prte_job_t); PMIX_LOAD_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace); prte_set_job_data_object(jdata); /* set the schizo personality to "prte" by default */ jdata->schizo = (struct prte_schizo_base_module_t*)prte_schizo_base_detect_proxy("prte"); if (NULL == jdata->schizo) { pmix_show_help("help-schizo-base.txt", "no-proxy", true, prte_tool_basename, "prte"); error = "select personality"; ret = PRTE_ERR_SILENT; goto error; } /* every job requires at least one app */ app = PMIX_NEW(prte_app_context_t); pmix_pointer_array_set_item(jdata->apps, 0, app); jdata->num_apps++; /* create and store a proc object for us */ proc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&proc->name, PRTE_PROC_MY_NAME->nspace, PRTE_PROC_MY_NAME->rank); proc->pid = prte_process_info.pid; proc->state = PRTE_PROC_STATE_RUNNING; pmix_pointer_array_set_item(jdata->procs, proc->name.rank, proc); /* record that the daemon job is running */ jdata->num_procs = 1; jdata->state = PRTE_JOB_STATE_RUNNING; /* obviously, we have "reported" */ jdata->num_reported = 1; /* setup my session directory here */ PMIX_OUTPUT_VERBOSE( (2, prte_ess_base_framework.framework_output, "%s setting up session dir with\n\ttmpdir: %s\n\thost %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == prte_process_info.tmpdir_base) ? "UNDEF" : prte_process_info.tmpdir_base, prte_process_info.nodename)); /* create the directory tree */ if (PRTE_SUCCESS != (ret = prte_session_dir(PRTE_PROC_MY_NAME))) { PRTE_ERROR_LOG(ret); error = "prte_session_dir"; goto error; } /* set the pmix_output env file location to be in the * proc-specific session directory. */ pmix_asprintf(&tmp, "%s/%s", jdata->session_dir, PMIX_RANK_PRINT(PRTE_PROC_MY_NAME->rank)); pmix_output_set_output_file_info(tmp, "output-", NULL, NULL); free(tmp); /* setup stdout/stderr */ if (prte_debug_daemons_file_flag) { /* if we are debugging to a file, then send stdout/stderr to * the prted log file */ /* define a log file name in the session directory */ snprintf(log_file, PRTE_PATH_MAX, "output-prted-%s-%s.log", prte_process_info.myproc.nspace, prte_process_info.nodename); log_path = pmix_os_path(false, prte_process_info.top_session_dir, log_file, NULL); fd = open(log_path, O_RDWR | O_CREAT | O_TRUNC, 0640); free(log_path); log_path = NULL; if (fd < 0) { /* couldn't open the file for some reason, so * just connect everything to /dev/null */ fd = open("/dev/null", O_RDWR | O_CREAT | O_TRUNC, 0666); if (fd < 0) { // cannot proceed ret = PRTE_ERROR; error = "open dev/null"; goto error; } } dup2(fd, STDOUT_FILENO); dup2(fd, STDERR_FILENO); if (fd != STDOUT_FILENO && fd != STDERR_FILENO) { close(fd); } } /* setup the PMIx server - we need this here in case the * communications infrastructure wants to register * information */ if (PRTE_SUCCESS != (ret = pmix_server_init())) { /* the server code already barked, so let's be quiet */ ret = PRTE_ERR_SILENT; error = "pmix_server_init"; goto error; } /* add network aliases to our list of alias hostnames - must * wait until after we init PMIx before getting them */ pmix_ifgetaliases(&prte_process_info.aliases); /* Setup the communication infrastructure */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_prtereachable_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_prtereachable_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_reachable_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_prtereachable_base_select"; goto error; } if (PRTE_SUCCESS != (ret = prte_rml_open())) { PRTE_ERROR_LOG(ret); error = "prte_rml_open"; goto error; } /* it is now safe to start the pmix server */ pmix_server_start(); /* select the errmgr */ if (PRTE_SUCCESS != (ret = prte_errmgr_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_errmgr_base_select"; goto error; } /* * Group communications */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_grpcomm_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_grpcomm_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_grpcomm_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_grpcomm_base_select"; goto error; } /* Open/select the odls */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_odls_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_odls_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_odls_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_odls_base_select"; goto error; } if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_rmaps_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_rmaps_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_rmaps_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_rmaps_base_select"; goto error; } /* if a topology file was given, then the rmaps framework open * will have reset our topology. Ensure we always get the right * one by setting our node topology afterwards */ t = PMIX_NEW(prte_topology_t); t->topo = prte_hwloc_topology; /* save the topology - note that this may have to be moved later * to ensure a common array position with the DVM master */ t->index = pmix_pointer_array_add(prte_node_topologies, t); if (15 < pmix_output_get_verbosity(prte_ess_base_framework.framework_output)) { char *output = NULL; pmix_topology_t topo; pmix_output(0, "%s Topology Info:", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); topo.source = "hwloc"; topo.topology = prte_hwloc_topology; ret = PMIx_Data_print(&output, NULL, &topo, PMIX_TOPO); if (PMIX_SUCCESS == ret) { fprintf(stderr, "%s\n", output); free(output); } else { PMIX_ERROR_LOG(ret); } } /* Now provide a chance for the PLM * to perform any module-specific init functions. This * needs to occur AFTER the communications are setup * as it may involve starting a non-blocking recv * Do this only if a specific PLM was given to us - the * prted has no need of the proxy PLM at all */ if (plm_in_use) { if (PRTE_SUCCESS != (ret = prte_plm.init())) { PRTE_ERROR_LOG(ret); error = "prte_plm_init"; goto error; } } /* setup I/O forwarding system - must come after we init routes */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_iof_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_iof_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_iof_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_iof_base_select"; goto error; } /* setup the FileM */ if (PRTE_SUCCESS != (ret = pmix_mca_base_framework_open(&prte_filem_base_framework, PMIX_MCA_BASE_OPEN_DEFAULT))) { PRTE_ERROR_LOG(ret); error = "prte_filem_base_open"; goto error; } if (PRTE_SUCCESS != (ret = prte_filem_base_select())) { PRTE_ERROR_LOG(ret); error = "prte_filem_base_select"; goto error; } return PRTE_SUCCESS; error: pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); if (NULL != jdata) { PMIX_RELEASE(jdata); } return PRTE_ERR_SILENT; } int prte_ess_base_prted_finalize(void) { prte_ess_base_signal_t *sig; unsigned int i; if (signals_set) { prte_event_del(&epipe_handler); prte_event_del(&term_handler); prte_event_del(&int_handler); /** Remove the USR signal handlers */ i = 0; PMIX_LIST_FOREACH(sig, &prte_ess_base_signals, prte_ess_base_signal_t) { prte_event_signal_del(forward_signals_events + i); ++i; } free(forward_signals_events); forward_signals_events = NULL; signals_set = false; } if (NULL != prte_errmgr.finalize) { prte_errmgr.finalize(); } /* close frameworks */ (void) pmix_mca_base_framework_close(&prte_filem_base_framework); (void) pmix_mca_base_framework_close(&prte_grpcomm_base_framework); (void) pmix_mca_base_framework_close(&prte_iof_base_framework); /* first stage shutdown of the errmgr, deregister the handler but keep * the required facilities until the rml and oob are offline */ (void) pmix_mca_base_framework_close(&prte_plm_base_framework); /* make sure our local procs are dead */ prte_odls.kill_local_procs(NULL); (void) pmix_mca_base_framework_close(&prte_odls_base_framework); (void) pmix_mca_base_framework_close(&prte_errmgr_base_framework); prte_rml_close(); (void) pmix_mca_base_framework_close(&prte_prtereachable_base_framework); (void) pmix_mca_base_framework_close(&prte_state_base_framework); /* shutdown the pmix server */ pmix_server_finalize(); return PRTE_SUCCESS; } static void shutdown_signal(int fd, short flags, void *arg) { PRTE_HIDE_UNUSED_PARAMS(fd, flags, arg); /* trigger the call to shutdown callback to protect * against race conditions - the trigger event will * check the one-time lock */ PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); } /** * Deal with sigpipe errors */ static void epipe_signal_callback(int fd, short flags, void *arg) { PRTE_HIDE_UNUSED_PARAMS(fd, flags, arg); /* for now, we just ignore them */ return; } /* Pass user signals to the local application processes */ static void signal_forward_callback(int fd, short event, void *arg) { prte_event_t *signal = (prte_event_t *) arg; int32_t signum, rc; pmix_data_buffer_t *cmd; prte_daemon_cmd_flag_t command = PRTE_DAEMON_SIGNAL_LOCAL_PROCS; PRTE_HIDE_UNUSED_PARAMS(fd, event); signum = PRTE_EVENT_SIGNAL(signal); if (!prte_execute_quiet) { fprintf(stderr, "PRTE: Forwarding signal %d to job\n", signum); } PMIX_DATA_BUFFER_CREATE(cmd); /* pack the command */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, cmd, &command, 1, PRTE_DAEMON_CMD); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(cmd); return; } /* pack the jobid */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, cmd, &PRTE_JOBID_WILDCARD, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(cmd); return; } /* pack the signal */ rc = PMIx_Data_pack(PRTE_PROC_MY_NAME, cmd, &PRTE_JOBID_WILDCARD, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(cmd); return; } /* send it to ourselves */ PRTE_RML_SEND(rc, PRTE_PROC_MY_NAME->rank, cmd, PRTE_RML_TAG_DAEMON); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(cmd); } } prrte-4.1.1~rc2/src/mca/ess/base/ess_base_std_prolog.c0000664000175000017500000000362315203542132023062 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include "src/mca/errmgr/errmgr.h" #include "src/runtime/prte_wait.h" #include "src/runtime/runtime_internals.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" int prte_ess_base_std_prolog(void) { int ret; char *error = NULL; /* Initialize the PRTE data type support */ if (PRTE_SUCCESS != (ret = prte_dt_init())) { error = "prte_dt_init"; goto error; } /* * Setup the waitpid/sigchld system */ if (PRTE_SUCCESS != (ret = prte_wait_init())) { PRTE_ERROR_LOG(ret); error = "prte_wait_init"; goto error; } return PRTE_SUCCESS; error: pmix_show_help("help-prte-runtime", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); return ret; } prrte-4.1.1~rc2/src/mca/ess/pals/0000775000175000017500000000000015203542132016717 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/ess/pals/ess_pals.h0000664000175000017500000000303715203542132020704 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * Copyright (c) 2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_ESS_PALS_H #define PRTE_ESS_PALS_H BEGIN_C_DECLS PRTE_MODULE_EXPORT extern prte_ess_base_component_t prte_mca_ess_pals_component; /* * Module open / close */ int prte_mca_ess_pals_component_open(void); int prte_mca_ess_pals_component_close(void); int prte_mca_ess_pals_component_query(pmix_mca_base_module_t **module, int *priority); END_C_DECLS #endif /* PRTE_ESS_PALS_H */ prrte-4.1.1~rc2/src/mca/ess/pals/ess_pals_component.c0000664000175000017500000000660015203542132022760 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include "src/util/proc_info.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/pals/ess_pals.h" extern prte_ess_base_module_t prte_ess_pals_module; /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_ess_base_component_t prte_mca_ess_pals_component = { PRTE_ESS_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "pals", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = prte_mca_ess_pals_component_open, .pmix_mca_close_component = prte_mca_ess_pals_component_close, .pmix_mca_query_component = prte_mca_ess_pals_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, ess, pals) int prte_mca_ess_pals_component_open(void) { return PRTE_SUCCESS; } int prte_mca_ess_pals_component_query(pmix_mca_base_module_t **module, int *priority) { /* Were we lauched by the PALS aprun launcher? Were * we given a path back to the HNP? If the * answer to both is "yes", then we were launched * by the PALS variant of aprun, so make ourselves available */ if (PRTE_PROC_IS_DAEMON && NULL != getenv("PALS_APID") && NULL != prte_process_info.my_hnp_uri) { *priority = 50; *module = (pmix_mca_base_module_t *) &prte_ess_pals_module; return PRTE_SUCCESS; } /* Sadly, no */ *priority = -1; *module = NULL; return PRTE_ERROR; } int prte_mca_ess_pals_component_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/pals/configure.m40000664000175000017500000000316115203542132021143 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2023 Triad National Security, LLC. All rights # reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_ess_pals_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_ess_pals_CONFIG],[ AC_CONFIG_FILES([src/mca/ess/pals/Makefile]) PRTE_CHECK_PALS([ess_pals], [ess_pals_good=1], [ess_pals_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$ess_pals_good" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([ess_pals_CPPFLAGS]) AC_SUBST([ess_pals_LDFLAGS]) AC_SUBST([ess_pals_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/ess/pals/Makefile.am0000664000175000017500000000350315203542132020754 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022 Nanook Consulting. All rights reserved. # Copyright (c) 2023 Triad National Security, LLC. All rights # reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ ess_pals.h \ ess_pals_component.c \ ess_pals_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_ess_pals_DSO component_noinst = component_install = prte_mca_ess_pals.la else component_noinst = libprtemca_ess_pals.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_ess_pals_la_SOURCES = $(sources) prte_mca_ess_pals_la_LDFLAGS = -module -avoid-version prte_mca_ess_pals_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_ess_pals_la_SOURCES =$(sources) libprtemca_ess_pals_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/ess/pals/ess_pals_module.c0000664000175000017500000001001715203542132022240 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * Copyright (c) 2023 Triad National Security, LLC. All rights * reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include #include "src/class/pmix_pointer_array.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/base/base.h" #include "src/mca/ess/ess.h" #include "src/mca/ess/pals/ess_pals.h" static int pals_set_name(void); static int rte_init(int argc, char **argv); static int rte_finalize(void); prte_ess_base_module_t prte_ess_pals_module = { .init = rte_init, .finalize = rte_finalize }; static int rte_init(int argc, char **argv) { int ret; char *error = NULL; PRTE_HIDE_UNUSED_PARAMS(argc, argv); /* run the prolog */ if (PRTE_SUCCESS != (ret = prte_ess_base_std_prolog())) { error = "prte_ess_base_std_prolog"; goto error; } /* Start by getting a unique name */ pals_set_name(); if (PRTE_SUCCESS != (ret = prte_ess_base_prted_setup())) { PRTE_ERROR_LOG(ret); error = "prte_ess_base_prted_setup"; goto error; } return PRTE_SUCCESS; error: if (PRTE_ERR_SILENT != ret && !prte_report_silent_errors) { pmix_show_help("help-prte-runtime.txt", "prte_init:startup:internal-failure", true, error, PRTE_ERROR_NAME(ret), ret); } return ret; } static int rte_finalize(void) { int ret; if (PRTE_SUCCESS != (ret = prte_ess_base_prted_finalize())) { PRTE_ERROR_LOG(ret); } return ret; } static int pals_set_name(void) { int pals_nodeid; pmix_rank_t vpid; char *tmp; PMIX_OUTPUT_VERBOSE((1, prte_ess_base_framework.framework_output, "ess:pals setting name")); if (NULL == prte_ess_base_nspace) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } PMIX_LOAD_NSPACE(PRTE_PROC_MY_NAME->nspace, prte_ess_base_nspace); if (NULL == prte_ess_base_vpid) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } vpid = strtoul(prte_ess_base_vpid, NULL, 10); if (NULL != getenv("PALS_NODEID")) { /* fix up the vpid and make it the "real" vpid */ pals_nodeid = atoi(getenv("PALS_NODEID")); PRTE_PROC_MY_NAME->rank = vpid + pals_nodeid; } else { return PRTE_ERR_NOT_FOUND; } PMIX_OUTPUT_VERBOSE((1, prte_ess_base_framework.framework_output, "ess:pals set name to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* get the num procs as provided in the cmd line param */ prte_process_info.num_daemons = prte_ess_base_num_procs; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/ess/ess.h0000664000175000017500000000456115203542132016731 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2012-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2018-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The PRTE Environment-Specific Services * */ #ifndef PRTE_ESS_H #define PRTE_ESS_H #include "prte_config.h" #include "types.h" #include "src/hwloc/hwloc-internal.h" #include "src/pmix/pmix-internal.h" #include "src/mca/mca.h" #include "src/runtime/runtime.h" #include "src/util/proc_info.h" BEGIN_C_DECLS /* * API functions */ /* * Initialize the RTE for this environment */ typedef int (*prte_ess_base_module_init_fn_t)(int argc, char **argv); /* * Finalize the RTE for this environment */ typedef int (*prte_ess_base_module_finalize_fn_t)(void); /* * the standard module data structure */ struct prte_ess_base_module_3_0_0_t { prte_ess_base_module_init_fn_t init; prte_ess_base_module_finalize_fn_t finalize; }; typedef struct prte_ess_base_module_3_0_0_t prte_ess_base_module_3_0_0_t; typedef struct prte_ess_base_module_3_0_0_t prte_ess_base_module_t; /* * the standard component data structure */ typedef pmix_mca_base_component_t prte_ess_base_component_t; /* * Macro for use in components that are of type ess */ #define PRTE_ESS_BASE_VERSION_3_0_0 PRTE_MCA_BASE_VERSION_3_0_0("ess", 3, 0, 0) /* Global structure for accessing ESS functions */ PRTE_EXPORT extern prte_ess_base_module_t prte_ess; /* holds selected module's function pointers */ END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/ess/Makefile.am0000664000175000017500000000243115203542132020014 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AM_CPPFLAGS = $(LTDLINCL) # main library setup noinst_LTLIBRARIES = libprtemca_ess.la libprtemca_ess_la_SOURCES = # pkgdata setup EXTRA_DIST = # local files headers = ess.h libprtemca_ess_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/iof/0000775000175000017500000000000015203542132015743 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/iof/iof_types.h0000664000175000017500000000313315203542132020115 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file */ #ifndef PRTE_IOF_TYPES_H #define PRTE_IOF_TYPES_H #include "prte_config.h" #include "types.h" BEGIN_C_DECLS /* Predefined tag values */ typedef uint16_t prte_iof_tag_t; #define PRTE_IOF_TAG_T PRTE_UINT16 #define PRTE_IOF_STDIN 0x0001 #define PRTE_IOF_STDOUT 0x0002 #define PRTE_IOF_STDERR 0x0004 #define PRTE_IOF_STDMERGE 0x0006 #define PRTE_IOF_STDDIAG 0x0008 #define PRTE_IOF_STDOUTALL 0x000e #define PRTE_IOF_STDALL 0x000f #define PRTE_IOF_EXCLUSIVE 0x0100 /* flow control flags */ #define PRTE_IOF_XON 0x1000 #define PRTE_IOF_XOFF 0x2000 /* tool requests */ #define PRTE_IOF_PULL 0x4000 #define PRTE_IOF_CLOSE 0x8000 END_C_DECLS #endif /* PRTE_IOF_TYPES_H */ prrte-4.1.1~rc2/src/mca/iof/hnp/0000775000175000017500000000000015203542132016530 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp.c0000664000175000017500000004414415203542132020325 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/util/pmix_output.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #ifdef HAVE_FCNTL_H # include #else # ifdef HAVE_SYS_FCNTL_H # include # endif #endif #include "src/event/event-internal.h" #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/mca/odls/odls_types.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "iof_hnp.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/base/iof_base_setup.h" /* LOCAL FUNCTIONS */ static void stdin_write_handler(int fd, short event, void *cbdata); /* API FUNCTIONS */ static int init(void); static int hnp_push(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd); static int hnp_pull(const pmix_proc_t *src_name, prte_iof_tag_t src_tag, int fd); static int hnp_close(const pmix_proc_t *peer, prte_iof_tag_t source_tag); static void hnp_complete(const prte_job_t *jdata); static int finalize(void); static int push_stdin(const pmix_proc_t *dst_name, uint8_t *data, size_t sz); /* The API's in this module are solely used to support LOCAL * procs - i.e., procs that are co-located to the HNP. Remote * procs interact with the HNP's IOF via the HNP's receive function, * which operates independently and is in the iof_hnp_receive.c file */ prte_iof_base_module_t prte_iof_hnp_module = { .init = init, .push = hnp_push, .pull = hnp_pull, .close = hnp_close, .complete = hnp_complete, .finalize = finalize, .push_stdin = push_stdin }; /* Initialize the module */ static int init(void) { /* post non-blocking recv to catch forwarded IO from * the orteds */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_IOF_HNP, PRTE_RML_PERSISTENT, prte_iof_hnp_recv, NULL); PMIX_CONSTRUCT(&prte_mca_iof_hnp_component.procs, pmix_list_t); return PRTE_SUCCESS; } /* Setup to read local data. */ static int hnp_push(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd) { prte_iof_proc_t *proct; int flags; /* don't do this if the dst vpid is invalid or the fd is negative! */ if (PMIX_RANK_INVALID == dst_name->rank || fd < 0) { return PRTE_SUCCESS; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:hnp pushing fd %d for process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), fd, PRTE_NAME_PRINT(dst_name))); /* do we already have this process in our list? */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, dst_name)) { /* found it */ goto SETUP; } } /* if we get here, then we don't yet have this proc in our list */ proct = PMIX_NEW(prte_iof_proc_t); PMIX_XFER_PROCID(&proct->name, dst_name); pmix_list_append(&prte_mca_iof_hnp_component.procs, &proct->super); SETUP: /* for stdout/stderr, set the file descriptor to non-blocking - do this before we setup * and activate the read event in case it fires right away */ if ((flags = fcntl(fd, F_GETFL, 0)) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_GETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } else { flags |= O_NONBLOCK; if (fcntl(fd, F_SETFL, flags) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_SETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } } /* define a read event and activate it */ if (src_tag & PRTE_IOF_STDOUT) { PRTE_IOF_READ_EVENT(&proct->revstdout, proct, fd, PRTE_IOF_STDOUT, prte_iof_hnp_read_local_handler, false); } else if (src_tag & PRTE_IOF_STDERR) { PRTE_IOF_READ_EVENT(&proct->revstderr, proct, fd, PRTE_IOF_STDERR, prte_iof_hnp_read_local_handler, false); } /* if -all- of the readevents for this proc have been defined, then * activate them. Otherwise, we can think that the proc is complete * because one of the readevents fires -prior- to all of them having * been defined! */ if (NULL != proct->revstdout && NULL != proct->revstderr) { if (!proct->revstdout->activated) { PRTE_IOF_READ_ACTIVATE(proct->revstdout); proct->revstdout->activated = true; } if (!proct->revstderr->activated) { PRTE_IOF_READ_ACTIVATE(proct->revstderr); proct->revstderr->activated = true; } } return PRTE_SUCCESS; } /* Push data to stdin of a client process * * (a) a specific name, usually vpid=0; or * * (b) all procs, specified by vpid=PRTE_VPID_WILDCARD * */ static int push_stdin(const pmix_proc_t *dst_name, uint8_t *data, size_t sz) { pmix_proc_t p; prte_iof_proc_t *proct; int rc; /* don't do this if the dst vpid is invalid */ if (PMIX_RANK_INVALID == dst_name->rank) { return PRTE_SUCCESS; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:hnp pushing stdin to process %s (size %zu)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(dst_name), sz)); /* if I am the DVM master and this is a wildcard name, then we have to * broadcast this to all daemons */ if (PMIX_RANK_WILDCARD == dst_name->rank) { PMIX_LOAD_PROCID(&p, PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD); rc = prte_iof_hnp_send_data_to_endpoint(&p, dst_name, PRTE_IOF_STDIN, data, sz); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); return rc; } } /* lookup the daemon hosting the target proc */ PMIX_LOAD_PROCID(&p, PRTE_PROC_MY_NAME->nspace, PMIX_RANK_WILDCARD); p.rank = prte_get_proc_daemon_vpid(dst_name); if (PMIX_RANK_INVALID == p.rank) { // proc must have finalized, so we can just ignore // it - no need to error log, this is just a // race condition return PRTE_ERR_ADDRESSEE_UNKNOWN; } /* if the host isn't me, send it to them */ if (p.rank != PRTE_PROC_MY_NAME->rank) { /* send the data to the daemon so it can * write it to the proc's fd - in this case, * we pass dst_name to indicate who is to * receive the data. If the connection closed, * numbytes will be zero so zero bytes will be * sent - this will tell the daemon to close * the fd for stdin to that proc */ rc = prte_iof_hnp_send_data_to_endpoint(&p, dst_name, PRTE_IOF_STDIN, data, sz); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } return rc; } /* local proc - see if we have this process in our list */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, dst_name)) { /* did they direct that the data go to this proc? */ if (NULL == proct->stdinev) { /* nope - ignore it */ continue; } /* send the bytes down the pipe - we even send 0 byte events * down the pipe so it forces out any preceding data before * closing the output stream */ if (NULL != proct->stdinev->wev) { if (PRTE_IOF_MAX_INPUT_BUFFERS < prte_iof_base_write_output(&proct->name, PRTE_IOF_STDIN, data, sz, proct->stdinev->wev)) { /* getting too backed up - stop the read event for now if it is still active */ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "buffer backed up - holding")); return PRTE_ERR_OUT_OF_RESOURCE; } } } } return PRTE_SUCCESS; } /* * Since we are the HNP, the only "pull" call comes from a local * process so we can record the file descriptor for its stdin. */ static int hnp_pull(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd) { prte_iof_proc_t *proct; int flags; /* this is a local call - only stdin is supported */ if (PRTE_IOF_STDIN != src_tag) { return PRTE_ERR_NOT_SUPPORTED; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:hnp pulling fd %d for process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), fd, PRTE_NAME_PRINT(dst_name))); /* set the file descriptor to non-blocking - do this before we setup * the sink in case it fires right away */ if ((flags = fcntl(fd, F_GETFL, 0)) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_GETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } else { flags |= O_NONBLOCK; if (fcntl(fd, F_SETFL, flags) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_SETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } } /* do we already have this process in our list? */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, dst_name)) { /* found it */ goto SETUP; } } /* if we get here, then we don't yet have this proc in our list */ proct = PMIX_NEW(prte_iof_proc_t); PMIX_XFER_PROCID(&proct->name, dst_name); pmix_list_append(&prte_mca_iof_hnp_component.procs, &proct->super); SETUP: PRTE_IOF_SINK_DEFINE(&proct->stdinev, dst_name, fd, PRTE_IOF_STDIN, stdin_write_handler); PMIX_XFER_PROCID(&proct->stdinev->daemon, PRTE_PROC_MY_NAME); PRTE_IOF_SINK_ACTIVATE(proct->stdinev->wev); return PRTE_SUCCESS; } /* * One of our local procs wants us to close the specifed * stream(s), thus terminating any potential io to/from it. */ static int hnp_close(const pmix_proc_t *peer, prte_iof_tag_t source_tag) { prte_iof_proc_t *proct; PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:hnp closing connection to process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(peer))); PMIX_LIST_FOREACH(proct, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, peer)) { if (PRTE_IOF_STDIN & source_tag) { if (NULL != proct->stdinev) { PMIX_RELEASE(proct->stdinev); } proct->stdinev = NULL; } if ((PRTE_IOF_STDOUT & source_tag) || (PRTE_IOF_STDMERGE & source_tag)) { if (NULL != proct->revstdout) { PMIX_RELEASE(proct->revstdout); } proct->revstdout = NULL; } if (PRTE_IOF_STDERR & source_tag) { if (NULL != proct->revstderr) { PMIX_RELEASE(proct->revstderr); } proct->revstderr = NULL; } /* if we closed them all, then remove this proc */ if (NULL == proct->stdinev && NULL == proct->revstdout && NULL == proct->revstderr) { pmix_list_remove_item(&prte_mca_iof_hnp_component.procs, &proct->super); PMIX_RELEASE(proct); } break; } } return PRTE_SUCCESS; } static void hnp_complete(const prte_job_t *jdata) { prte_iof_proc_t *proct, *next; /* cleanout any lingering sinks */ PMIX_LIST_FOREACH_SAFE(proct, next, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_NSPACE(jdata->nspace, proct->name.nspace)) { pmix_list_remove_item(&prte_mca_iof_hnp_component.procs, &proct->super); if (NULL != proct->revstdout) { PMIX_RELEASE(proct->revstdout); } proct->revstdout = NULL; if (NULL != proct->revstderr) { PMIX_RELEASE(proct->revstderr); } proct->revstderr = NULL; PMIX_RELEASE(proct); } } } static int finalize(void) { PMIX_DESTRUCT(&prte_mca_iof_hnp_component.procs); return PRTE_SUCCESS; } /* this function is called by the event library and thus * can access information global to the state machine */ static void stdin_write_handler(int fd, short event, void *cbdata) { prte_iof_sink_t *sink = (prte_iof_sink_t *) cbdata; prte_iof_write_event_t *wev = sink->wev; pmix_list_item_t *item; prte_iof_write_output_t *output; int num_written, total_written = 0; PRTE_HIDE_UNUSED_PARAMS(fd, event); PMIX_ACQUIRE_OBJECT(sink); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s hnp:stdin:write:handler writing %d data to %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int)pmix_list_get_size(&wev->outputs), wev->fd)); wev->pending = false; while (NULL != (item = pmix_list_remove_first(&wev->outputs))) { output = (prte_iof_write_output_t *) item; /* if an abnormal termination has occurred, just dump * this data as we are aborting */ if (prte_abnormal_term_ordered) { PMIX_RELEASE(output); continue; } if (0 == output->numbytes) { /* this indicates we are to close the fd - there is * nothing to write */ PMIX_OUTPUT_VERBOSE((20, prte_iof_base_framework.framework_output, "%s iof:hnp closing fd %d on write event due to zero bytes output", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); goto finish; } num_written = write(wev->fd, output->data, output->numbytes); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s hnp:stdin:write:handler wrote %d bytes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), num_written)); if (num_written < 0) { if (EAGAIN == errno || EINTR == errno) { /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* leave the write event running so it will call us again * when the fd is ready. */ goto re_enter; } /* otherwise, something bad happened so all we can do is declare an * error and abort */ PMIX_RELEASE(output); PMIX_OUTPUT_VERBOSE( (20, prte_iof_base_framework.framework_output, "%s iof:hnp closing fd %d on write event due to negative bytes written", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); goto finish; } else if (num_written < output->numbytes) { PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s hnp:stdin:write:handler incomplete write %d - adjusting data", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), num_written)); /* incomplete write - adjust data and count to avoid duplicate output */ output->numbytes -= num_written; memmove(output->data, &output->data[num_written], output->numbytes); /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* leave the write event running so it will call us again * when the fd is ready. */ goto re_enter; } PMIX_RELEASE(output); total_written += num_written; if ((PRTE_IOF_SINK_BLOCKSIZE <= total_written) && wev->always_writable) { goto re_enter; } } goto check; re_enter: PRTE_IOF_SINK_ACTIVATE(wev); check: if (sink->closed && 0 == pmix_list_get_size(&wev->outputs)) { /* the sink has already been closed and everything was written, time to release it */ PMIX_RELEASE(sink); } return; finish: PMIX_RELEASE(wev); sink->wev = NULL; return; } prrte-4.1.1~rc2/src/mca/iof/hnp/owner.txt0000664000175000017500000000024415203542132020423 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp_read.c0000664000175000017500000001254015203542132021313 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2018-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/odls/odls_types.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" #include "iof_hnp.h" static void lkcbfunc(pmix_status_t status, void *cbdata) { prte_iof_deliver_t *p = (prte_iof_deliver_t*)cbdata; /* nothing to do here - we use this solely to * ensure that IOF_deliver doesn't block */ if (PMIX_SUCCESS != status) { PMIX_ERROR_LOG(status); } PMIX_RELEASE(p); } /* this is the read handler for my own child procs. In this case, * the data is going nowhere - I just output it myself */ void prte_iof_hnp_read_local_handler(int fd, short event, void *cbdata) { prte_iof_read_event_t *rev = (prte_iof_read_event_t *) cbdata; unsigned char data[PRTE_IOF_BASE_MSG_MAX]; int32_t numbytes; prte_iof_proc_t *proct = (prte_iof_proc_t *) rev->proc; prte_iof_deliver_t *p; pmix_iof_channel_t pchan; pmix_status_t prc; PRTE_HIDE_UNUSED_PARAMS(event); PMIX_ACQUIRE_OBJECT(rev); /* As we may use timer events, fd can be bogus (-1) * use the right one here */ fd = rev->fd; /* read up to the fragment size */ memset(data, 0, PRTE_IOF_BASE_MSG_MAX); numbytes = read(fd, data, sizeof(data)); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s read %d bytes from %s of %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes, (PRTE_IOF_STDOUT & rev->tag) ? "stdout" : ((PRTE_IOF_STDERR & rev->tag) ? "stderr" : "stddiag"), PRTE_NAME_PRINT(&proct->name))); if (NULL == proct) { /* this is an error - nothing we can do */ PRTE_ERROR_LOG(PRTE_ERR_ADDRESSEE_UNKNOWN); return; } if (numbytes <= 0) { if (0 > numbytes) { /* either we have a connection error or it was a non-blocking read */ /* non-blocking, retry */ if (EAGAIN == errno || EINTR == errno) { PRTE_IOF_READ_ACTIVATE(rev); return; } } /* go down and close the fd etc */ goto CLEAN_RETURN; } /* this must be output from one of my local procs */ pchan = 0; if (PRTE_IOF_STDOUT & rev->tag) { pchan |= PMIX_FWD_STDOUT_CHANNEL; } if (PRTE_IOF_STDERR & rev->tag) { pchan |= PMIX_FWD_STDERR_CHANNEL; } if (PRTE_IOF_STDDIAG & rev->tag) { pchan |= PMIX_FWD_STDDIAG_CHANNEL; } /* setup the byte object */ p = PMIX_NEW(prte_iof_deliver_t); PMIX_XFER_PROCID(&p->source, &proct->name); p->bo.bytes = (char*)malloc(numbytes); memcpy(p->bo.bytes, data, numbytes); p->bo.size = numbytes; prc = PMIx_server_IOF_deliver(&p->source, pchan, &p->bo, NULL, 0, lkcbfunc, (void*)p); if (PMIX_SUCCESS != prc) { PMIX_ERROR_LOG(prc); PMIX_RELEASE(p); } /* re-add the event */ PRTE_IOF_READ_ACTIVATE(rev); return; CLEAN_RETURN: /* if we read 0 bytes from the stdout/err/diag, there is * nothing to output - release the appropriate event. * This will delete the read event and close the file descriptor */ /* make sure we don't do recursive delete on the proct */ PMIX_RETAIN(proct); if (rev->tag & PRTE_IOF_STDOUT) { PMIX_RELEASE(proct->revstdout); proct->revstdout = NULL; } else if (rev->tag & PRTE_IOF_STDERR) { PMIX_RELEASE(proct->revstderr); proct->revstderr = NULL; } /* check to see if they are all done */ if (NULL == proct->revstdout && NULL == proct->revstderr) { /* this proc's iof is complete */ PRTE_ACTIVATE_PROC_STATE(&proct->name, PRTE_PROC_STATE_IOF_COMPLETE); } PMIX_RELEASE(proct); return; } prrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp_component.c0000664000175000017500000000613115203542132022401 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/event/event-internal.h" #include "src/mca/base/pmix_base.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "iof_hnp.h" #include "src/mca/iof/base/base.h" /* * Local functions */ static int prte_iof_hnp_open(void); static int prte_iof_hnp_close(void); static int prte_iof_hnp_query(pmix_mca_base_module_t **module, int *priority); /* * Public string showing the iof hnp component version number */ const char *prte_mca_iof_hnp_component_version_string = "PRTE hnp iof MCA component version " PRTE_VERSION; prte_mca_iof_hnp_component_t prte_mca_iof_hnp_component = { .super = { PRTE_IOF_BASE_VERSION_2_0_0, .pmix_mca_component_name = "hnp", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open, close, and query functions */ .pmix_mca_open_component = prte_iof_hnp_open, .pmix_mca_close_component = prte_iof_hnp_close, .pmix_mca_query_component = prte_iof_hnp_query, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, iof, hnp) /** * component open/close/init function */ static int prte_iof_hnp_open(void) { /* Nothing to do */ return PRTE_SUCCESS; } static int prte_iof_hnp_close(void) { return PRTE_SUCCESS; } /** * Module query */ static int prte_iof_hnp_query(pmix_mca_base_module_t **module, int *priority) { /* if we are not the HNP, then don't use this module */ if (!PRTE_PROC_IS_MASTER && !PRTE_PROC_IS_MASTER) { *priority = -1; *module = NULL; return PRTE_ERROR; } *priority = 100; *module = (pmix_mca_base_module_t *) &prte_iof_hnp_module; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp.h0000664000175000017500000000610115203542132020321 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * The hnp IOF component is used in HNP processes only. It is the * "hub" for all IOF activity, meaning that *all* IOF traffic is * routed to the hnp component, and this component figures out where * it is supposed to go from there. Specifically: there is *no* * direct proxy-to-proxy IOF communication. If a proxy/orted wants to * get a stream from another proxy/orted, the stream will go * proxy/orted -> HNP -> proxy/orted. * * The hnp IOF component does two things: 1. forward fragments between * file descriptors and streams, and 2. maintain forwarding tables to * "route" incoming fragments to outgoing destinations (both file * descriptors and other published streams). * */ #ifndef PRTE_IOF_HNP_H #define PRTE_IOF_HNP_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif /* HAVE_SYS_TYPES_H */ #ifdef HAVE_SYS_UIO_H # include #endif /* HAVE_SYS_UIO_H */ #ifdef HAVE_NET_UIO_H # include #endif /* HAVE_NET_UIO_H */ #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" BEGIN_C_DECLS /** * IOF HNP Component */ struct prte_mca_iof_hnp_component_t { prte_iof_base_component_t super; pmix_list_t procs; prte_event_t stdinsig; }; typedef struct prte_mca_iof_hnp_component_t prte_mca_iof_hnp_component_t; PRTE_MODULE_EXPORT extern prte_mca_iof_hnp_component_t prte_mca_iof_hnp_component; extern prte_iof_base_module_t prte_iof_hnp_module; void prte_iof_hnp_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); void prte_iof_hnp_read_local_handler(int fd, short event, void *cbdata); void prte_iof_hnp_stdin_cb(int fd, short event, void *cbdata); bool prte_iof_hnp_stdin_check(int fd); int prte_iof_hnp_send_data_to_endpoint(const pmix_proc_t *host, const pmix_proc_t *target, prte_iof_tag_t tag, unsigned char *data, int numbytes); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/iof/hnp/Makefile.am0000664000175000017500000000345415203542132020572 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_iof_hnp_DSO component_noinst = component_install = prte_mca_iof_hnp.la else component_noinst = libprtemca_iof_hnp.la component_install = endif hnp_SOURCES = \ iof_hnp.c \ iof_hnp.h \ iof_hnp_component.c \ iof_hnp_read.c \ iof_hnp_send.c \ iof_hnp_receive.c mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_iof_hnp_la_SOURCES = $(hnp_SOURCES) prte_mca_iof_hnp_la_LDFLAGS = -module -avoid-version prte_mca_iof_hnp_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_iof_hnp_la_SOURCES = $(hnp_SOURCES) libprtemca_iof_hnp_la_LIBADD = libprtemca_iof_hnp_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp_receive.c0000664000175000017500000001247315203542132022027 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #ifdef HAVE_FCNTL_H # include #else # ifdef HAVE_SYS_FCNTL_H # include # endif #endif #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" #include "iof_hnp.h" static void lkcbfunc(pmix_status_t status, void *cbdata) { prte_iof_deliver_t *p = (prte_iof_deliver_t*)cbdata; /* nothing to do here - we use this solely to * ensure that IOF_deliver doesn't block */ if (PMIX_SUCCESS != status) { PMIX_ERROR_LOG(status); } PMIX_RELEASE(p); } void prte_iof_hnp_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { pmix_proc_t origin; prte_iof_tag_t stream; int32_t count, numbytes; int rc; prte_iof_proc_t *proct; pmix_iof_channel_t pchan; prte_iof_deliver_t *p; pmix_status_t prc; PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s received IOF msg from proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender))); /* unpack the stream first as this may be flow control info */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &stream, &count, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } /* get name of the process whose io we are discussing */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &origin, &count, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s received IOF cmd for source %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&origin))); /* this must have come from a daemon forwarding output - unpack the data */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &numbytes, &count, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } if (0 == numbytes) { /* nothing to do - shouldn't have been sent */ goto CLEAN_RETURN; } p = PMIX_NEW(prte_iof_deliver_t); PMIX_XFER_PROCID(&p->source, &origin); p->bo.bytes = (char*)malloc(numbytes); rc = PMIx_Data_unpack(NULL, buffer, p->bo.bytes, &numbytes, PMIX_BYTE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(p); goto CLEAN_RETURN; } p->bo.size = numbytes; PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s unpacked %d bytes from remote proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes, PRTE_NAME_PRINT(&origin))); /* do we already have this process in our list? */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_hnp_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, &origin)) { /* found it */ goto NSTEP; } } /* if we get here, then we don't yet have this proc in our list */ proct = PMIX_NEW(prte_iof_proc_t); PMIX_XFER_PROCID(&proct->name, &origin); pmix_list_append(&prte_mca_iof_hnp_component.procs, &proct->super); NSTEP: pchan = 0; if (PRTE_IOF_STDOUT & stream) { pchan |= PMIX_FWD_STDOUT_CHANNEL; } if (PRTE_IOF_STDERR & stream) { pchan |= PMIX_FWD_STDERR_CHANNEL; } if (PRTE_IOF_STDDIAG & stream) { pchan |= PMIX_FWD_STDDIAG_CHANNEL; } /* output this thru our PMIx server */ prc = PMIx_server_IOF_deliver(&p->source, pchan, &p->bo, NULL, 0, lkcbfunc, (void*)p); if (PMIX_SUCCESS != prc) { PMIX_ERROR_LOG(prc); PMIX_RELEASE(p); } CLEAN_RETURN: return; } prrte-4.1.1~rc2/src/mca/iof/hnp/iof_hnp_send.c0000664000175000017500000000757415203542132021344 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012 Los Alamos National Security, LLC * All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/rml/rml.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" #include "iof_hnp.h" int prte_iof_hnp_send_data_to_endpoint(const pmix_proc_t *host, const pmix_proc_t *target, prte_iof_tag_t tag, unsigned char *data, int numbytes) { pmix_data_buffer_t *buf; int rc; /* if the host is a daemon and we are in the process of aborting, * then ignore this request. We leave it alone if the host is not * a daemon because it might be a tool that wants to watch the * output from an abort procedure */ if (PMIX_CHECK_NSPACE(PRTE_JOB_FAMILY_PRINT(host->nspace), PRTE_JOB_FAMILY_PRINT(PRTE_PROC_MY_NAME->nspace)) && prte_dvm_abort_ordered) { return PRTE_SUCCESS; } PMIX_DATA_BUFFER_CREATE(buf); /* pack the tag - we do this first so that flow control messages can * consist solely of the tag */ rc = PMIx_Data_pack(NULL, buf, &tag, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } /* pack the name of the target - this is either the intended * recipient (if the tag is stdin and we are sending to a daemon), * or the source (if we are sending to anyone else) */ rc = PMIx_Data_pack(NULL, buf, (void*)target, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } /* pack the data - if numbytes is zero, we will pack zero bytes */ rc = PMIx_Data_pack(NULL, buf, data, numbytes, PMIX_BYTE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } /* if the target is wildcard, then this needs to go to everyone - xcast it */ if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, host->nspace) && PMIX_RANK_WILDCARD == host->rank) { /* xcast this to everyone - the local daemons will know how to handle it */ (void) prte_grpcomm.xcast(PRTE_RML_TAG_IOF_PROXY, buf); PMIX_DATA_BUFFER_RELEASE(buf); return PRTE_SUCCESS; } /* send the buffer to the host - this is either a daemon or * a tool that requested IOF */ PRTE_RML_SEND(rc, host->rank, buf, PRTE_RML_TAG_IOF_PROXY); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/iof/README.txt0000664000175000017500000002106215203542132017442 0ustar alastairalastairSome notes from Jeff/Rolf while trying to figure out how IOF works... 1. E-mail from Rolf->Jeff with some pretty ASCII art 2. Notes from Jeff about problems we'll have when/if we ever try to use IOF more creatively. =============================================================================== From: Rolf.Vandevaart@Sun.COM Subject: Picture of IOF side of things. Date: June 7, 2007 10:43:42 AM EDT To: jsquyres@cisco.com Not sure if this will come out for you, but here is an ASCII represantation of what the HNP looks like after IOF has been wired up. VIEW OF DATA STRUCTURES ON IOF SVC SIDE KEY: PRTE_IOF_SOURCE=0 PRTE_IOF_SINK=1 After the job starts up, this is how things look on the HNP side of things. ENDPOINTS (prte_iof_base_endpoint_t) mode origin tag fd seq ack src_frags sink_frags notes ============================================================================================= 1 0,0,0 1 1 0 0 0 0 pull() call from rmgr<- ============================================================================================= | 1 0,0,0 2 2 0 0 0 0 pull() call from rmgr<-|--- ============================================================================================= | | 0 0,0,0 0 0 0 0 0 0 push() call from rmgr | | ============================================================================================= | | | | | | SUBSCRIBERS (prte_iof_svc_sub_t) | | ORIGIN TARGET (list) | | name mask tag name mask tag endpoint forward has_been_acked last_ack_forwarded | | ====================================================================================== | | 0,1,0 2 1 0,0,0 2 1 0 | | ptr-------------------------------------------------| | ====================================================================================== | 0,1,0 2 2 0,0,0 2 2 0 | ptr-----------------------------------------------------| ====================================================================================== 0,0,0 ff 0 0,1,0 2 0 NULL 1 ----------------------------------- | ====================================================================================== | | | PUBLISHED (prte_iof_svc_pub_t) | name proxy mask tag endpoint | ========================================================================================= | 0,1,0 0,0,1 ff 0 NULL <------------------ ========================================================================================= FORWARD (prte_iof_svc_fwd_t) This structure is just a connection from a subscriber to publisher. I have omitted it in the drawings. However, it is worth pointing out the structure as I am not clear on why we have the table. struct prte_iof_svc_fwd_t { pmix_list_item_t super; prte_iof_svc_pub_t* fwd_pub; pmix_hash_table_t fwd_seq_hash; }; Note: This first subscriber says that it will receive from any process in the job. Note that the jobid=1 and the mask=2. So, we expect this to collect the stdout from any of the ranks. Obviously the second subscriber says the same thing but for stderr. The third subscriber is for receving data from stdin and sending it out to rank 0 of the job. Notice the mask=ff which means compare jobid,vpid when addressing where the data goes. The first endpoint is created by a call to pull by the rmgr. After the endpoint is created, a subscription is created as well. Then, the subscription is tied to the endpoint. For the stdin creation, we first create the subscription, and then the endpoint. In that way, the endpoint is not found and does not get tied to the subscription. Hmmm, this I do not really understand. APPENDIX A These are the defines that go with the mask. #define PRTE_NS_CMP_NONE 0x00 #define PRTE_NS_CMP_JOBID 0x02 #define PRTE_NS_CMP_VPID 0x04 #define PRTE_NS_CMP_ALL 0Xff When we get a HDR_MSG, we call prte_iof_svc_proxy_msg() APPENDIX B There are two dbx files that help get to where we want to get for seeing how things work. start.x : Run this first to get initial breakpoint. Needs this so we can set additional breakpoints. This also has some very helpful aliases for looking at the structures shown above. follow.x : Run this second to set initial breakpoints and setup some useful aliases. =============================================================================== Random notes from Jeff: - Many issues may not come up because we only have single subscribers; I'm sure new things will come up. Examples: - What happens if all subscribers to a stream disconnect, and then a new subscriber connects? I'm guessing the ACKs will be all screwed up and we'll never end up reading from that fd again (because it will likely be stalled because not enough acks have been received, and therefore it removed itself from the event engine). - If all subscribers disconnect from a stdin/SINK, chances are that we'll lose the last frag that was sent before the disconnect. I.e., if there was a frag in flight when the disconnect was received, that frag is effectively lost. So if someone reconnects to the stdin stream later, it won't start reading exactly where the first subscriber left off. We need to define what is *supposed* to happen here... - odls default: make handling of vpid 0 uniform between setup and takedown -- some kind of global variable, perhaps? (not multi-proc / thread safe) - odls default: currently, we publish stdin (if relevant), stdout, and stderr (note that only the stdin publish message gets sent to svc; the publish for SOURCEs stdout/stderr is not actually sent to the svc because all SOURCE frags are sent to the svc automatically). But we only unpublish stdout. I think we should either: - publish stdin, stdout, stderr, and unpublish stdin, stdout, stderr or - publish stdin, and unpublish stdin I.e., make the code symmetric. Note, however, that unpublish for STDOUT/STDERR are sent to the svc (whereas publish for STDOUT/STDERR are not). So if we unpublish stdout/stderr, we'll be creating a storm to the svc upon shutdown (i.e,. scalability problems). :-( - for scalability, we want to be able to change the proxy to *not* unconfitionally send everything to svc. But this has the problem that if we do this, then we have to send the publish request to the svc (which we don't today since everything just automatically goes to svc). But then in the common case (where vpid!=0 has no stdout/stderr), we're flooding svc with N publish requests from all the vpids, simply creating a different scalability problem (during startup). - random q: are the proxy publish requests not sent back to svc because it prevents a storm of publish requests during startup? I.e., this was intentional to give better scalability? Could be; but it still seems weird... Perhaps a better scheme would be to have the IOF *assume* that the stdin/stdout/stderr are all published upon startup (or be told by a single control message; perhaps in the app context?) and further *assume* that they are all unpublished when the job completes. Putting this info in the app context (for example) might jive with a more capable prun that allows flexible stdin/stdout/stderr mapping (think: mpirun --screen ...). mpirun makes the decision about how to wire up stdin/stdout/stderr and includes it in the app context (or whatever). This is given to the svc who then creates publications as relevant. Upon job completion, all publications/subscriptions related to that job are destroyed. prrte-4.1.1~rc2/src/mca/iof/iof.h0000664000175000017500000001704115203542132016674 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * I/O Forwarding Service * The I/O forwarding service (IOF) is used to connect stdin, stdout, and * stderr file descriptor streams from MPI processes to the user * * The design is fairly simple: when a proc is spawned, the IOF establishes * connections between its stdin, stdout, and stderr to a * corresponding IOF stream. In addition, the IOF designates a separate * stream for passing OMPI/PRTE internal diagnostic/help output to mpirun. * This is done specifically to separate such output from the user's * stdout/err - basically, it allows us to present it to the user in * a separate format for easier recognition. Data read from a source * on any stream (e.g., printed to stdout by the proc) is relayed * by the local daemon to the other end of the stream - i.e., stdin * is relayed to the local proc, while stdout/err is relayed to mpirun. * Thus, the eventual result is to connect ALL streams to/from * the application process and mpirun. * * Note: By default, data read from stdin is forwarded -only- to rank=0. * Stdin for all other procs is tied to "/dev/null". * * External tools can "pull" copies of stdout/err and * the diagnostic stream from mpirun for any process. In this case, * mpirun will send a copy of the output to the "pulling" process. Note that external tools * cannot "push" something into stdin unless the user specifically directed * that stdin remain open, nor under any conditions "pull" a copy of the * stdin being sent to rank=0. * * Tools can exploit either of two mechanisms for this purpose: * * (a) call prte_init themselves and utilize the PRTE tool comm * library to access the IOF. This also provides access to * other tool library functions - e.g., to order that a job * be spawned; or * * (b) fork/exec the "prte-iof" tool and let it serve as the interface * to mpirun. This lets the tool avoid calling prte_init, and means * the tool will not have to compile against the PRTE/OMPI libraries. * However, the prte-iof tool is limited solely to interfacing * stdio and cannot be used for other functions included in * the tool comm library * * Thus, mpirun acts as a "switchyard" for IO, taking input from stdin * and passing it to rank=0 of the job, and taking stdout/err/diag from all * ranks and passing it to its own stdout/err/diag plus any "pull" * requestors. * * Streams are identified by PRTE process name (to include wildcards, * such as "all processes in PRTE job X") and tag. There are * currently only 4 allowed predefined tags: * * - PRTE_IOF_STDIN (value 0) * - PRTE_IOF_STDOUT (value 1) * - PRTE_IOF_STDERR (value 2) * - PRTE_IOF_INTERNAL (value 3): for "internal" messages * from the infrastructure, just to differentiate them from user job * stdout/stderr * * Note that since streams are identified by PRTE process name, the * caller has no idea whether the stream is on the local node or a * remote node -- it's just a stream. * * IOF components are selected on a "one of many" basis, meaning that * only one IOF component will be selected for a given process. * Details for the various components are given in their source code * bases. * * Each IOF component must support the following API: * * push: Tie a local file descriptor (*not* a stream!) to the stdin * of the specified process. If the user has not specified that stdin * of the specified process is to remain open, this will return an error. * * pull: Tie a local file descriptor (*not* a stream!) to a stream. * Subsequent input that appears via the stream will * automatically be sent to the target file descriptor until the * stream is "closed" or an EOF is received on the local file descriptor. * Valid source values include PRTE_IOF_STDOUT, PRTE_IOF_STDERR, and * PRTE_IOF_INTERNAL * * close: Closes a stream, flushing any pending data down it and * terminating any "push/pull" connections against it. Unclear yet * if this needs to be blocking, or can be done non-blocking. * * flush: Block until all pending data on all open streams has been * written down local file descriptors and/or completed sending across * the OOB to remote process targets. * */ #ifndef PRTE_IOF_H #define PRTE_IOF_H #include "prte_config.h" #include "types.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "iof_types.h" BEGIN_C_DECLS /* Initialize the selected module */ typedef int (*prte_iof_base_init_fn_t)(void); /** * Explicitly push data from the specified input file descriptor to * the stdin of the indicated peer(s). The provided peer name can * include wildcard values. * * @param peer Name of target peer(s) * @param fd Local file descriptor for input. */ typedef int (*prte_iof_base_push_fn_t)(const pmix_proc_t *peer, prte_iof_tag_t src_tag, int fd); /** * Explicitly pull data from the specified set of SOURCE peers and * dump to the indicated output file descriptor. Any fragments that * arrive on the stream will automatically be written down the fd. * * @param peer Name used to qualify set of origin peers. * @param source_tag Indicates the output streams to be forwarded * @param fd Local file descriptor for output. */ typedef int (*prte_iof_base_pull_fn_t)(const pmix_proc_t *peer, prte_iof_tag_t source_tag, int fd); /** * Close the specified iof stream(s) from the indicated peer(s) */ typedef int (*prte_iof_base_close_fn_t)(const pmix_proc_t *peer, prte_iof_tag_t source_tag); typedef int (*prte_iof_base_push_stdin_fn_t)(const pmix_proc_t *dst_name, uint8_t *data, size_t sz); /* Flag that a job is complete */ typedef void (*prte_iof_base_complete_fn_t)(const prte_job_t *jdata); /* finalize the selected module */ typedef int (*prte_iof_base_finalize_fn_t)(void); /** * IOF module. */ struct prte_iof_base_module_2_0_0_t { prte_iof_base_init_fn_t init; prte_iof_base_push_fn_t push; prte_iof_base_pull_fn_t pull; prte_iof_base_close_fn_t close; prte_iof_base_complete_fn_t complete; prte_iof_base_finalize_fn_t finalize; prte_iof_base_push_stdin_fn_t push_stdin; }; typedef struct prte_iof_base_module_2_0_0_t prte_iof_base_module_2_0_0_t; typedef prte_iof_base_module_2_0_0_t prte_iof_base_module_t; PRTE_EXPORT extern prte_iof_base_module_t prte_iof; typedef pmix_mca_base_component_t prte_iof_base_component_t; END_C_DECLS /* * Macro for use in components that are of type iof */ #define PRTE_IOF_BASE_VERSION_2_0_0 PRTE_MCA_BASE_VERSION_3_0_0("iof", 2, 0, 0) #endif /* PRTE_IOF_H */ prrte-4.1.1~rc2/src/mca/iof/base/0000775000175000017500000000000015203542132016655 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/iof/base/iof_base_setup.h0000664000175000017500000000335115203542132022017 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #ifndef IOF_BASE_SETUP_H_ #define IOF_BASE_SETUP_H_ #include "prte_config.h" #include "types.h" #include "src/mca/iof/base/base.h" struct prte_iof_base_io_conf_t { int usepty; bool connect_stdin; /* private - callers should not modify these fields */ int p_stdin[2]; int p_stdout[2]; int p_stderr[2]; }; typedef struct prte_iof_base_io_conf_t prte_iof_base_io_conf_t; /** * Do pre-fork IOF setup tasks * * Do all stdio forwarding that must be done before fork() is called. * This might include creating pipes or ptys or similar work. */ PRTE_EXPORT int prte_iof_base_setup_prefork(prte_iof_base_io_conf_t *opts); PRTE_EXPORT int prte_iof_base_setup_child(prte_iof_base_io_conf_t *opts, char ***env); PRTE_EXPORT int prte_iof_base_setup_parent(const pmix_proc_t *name, prte_iof_base_io_conf_t *opts); #endif prrte-4.1.1~rc2/src/mca/iof/base/iof_base_setup.c0000664000175000017500000001742515203542132022021 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2017-2021 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif #include #include #ifdef HAVE_SYS_WAIT_H # include #endif #include #ifdef HAVE_UTIL_H # include #endif #ifdef HAVE_PTY_H # include #endif #ifdef HAVE_FCNTL_H # include #endif #ifdef HAVE_TERMIOS_H # include # ifdef HAVE_TERMIO_H # include # endif #endif #ifdef HAVE_LIBUTIL_H # include #endif #ifdef HAVE_SYS_IOCTL_H # include #endif #include "src/mca/errmgr/errmgr.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/name_fns.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_pty.h" #include "src/util/pmix_show_help.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/base/iof_base_setup.h" #include "src/mca/iof/iof.h" int prte_iof_base_setup_prefork(prte_iof_base_io_conf_t *opts) { int ret = -1; fflush(stdout); /* first check to make sure we can do ptys */ #if PRTE_ENABLE_PTY_SUPPORT if (opts->usepty) { struct winsize *wp = NULL; /** * It has been reported that on MAC OS X 10.4 and prior one cannot * safely close the writing side of a pty before completly reading * all data inside. * There seems to be two issues: first all pending data is * discarded, and second it randomly generate kernel panics. * Apparently this issue was fixed in 10.5 so by now we use the * pty exactly as we use the pipes. * This comment is here as a reminder. */ #ifdef TIOCGWINSZ struct winsize ws; if (0 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)) { wp = &ws; } #endif ret = pmix_openpty(&(opts->p_stdout[0]), &(opts->p_stdout[1]), (char *) NULL, (struct termios *) NULL, wp); } #else opts->usepty = 0; #endif if (ret < 0) { opts->usepty = 0; if (pipe(opts->p_stdout) < 0) { PMIX_ERROR_LOG(PMIX_ERR_SYS_LIMITS_PIPES); return PMIX_ERR_SYS_LIMITS_PIPES; } } if (opts->connect_stdin) { if (pipe(opts->p_stdin) < 0) { PMIX_ERROR_LOG(PMIX_ERR_SYS_LIMITS_PIPES); return PMIX_ERR_SYS_LIMITS_PIPES; } } if (pipe(opts->p_stderr) < 0) { PMIX_ERROR_LOG(PMIX_ERR_SYS_LIMITS_PIPES); return PMIX_ERR_SYS_LIMITS_PIPES; } return PRTE_SUCCESS; } int prte_iof_base_setup_child(prte_iof_base_io_conf_t *opts, char ***env) { int ret; PRTE_HIDE_UNUSED_PARAMS(env); if (opts->connect_stdin) { close(opts->p_stdin[1]); } close(opts->p_stdout[0]); close(opts->p_stderr[0]); if (opts->usepty) { /* disable echo */ struct termios term_attrs; if (tcgetattr(opts->p_stdout[1], &term_attrs) < 0) { return PMIX_ERR_PIPE_SETUP_FAILURE; } term_attrs.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHOCTL | ECHOKE | ECHONL); term_attrs.c_iflag &= ~(ICRNL | INLCR | ISTRIP | INPCK | IXON); term_attrs.c_oflag &= ~( #ifdef OCRNL /* OS X 10.3 does not have this value defined */ OCRNL | #endif ONLCR); if (tcsetattr(opts->p_stdout[1], TCSANOW, &term_attrs) == -1) { return PMIX_ERR_PIPE_SETUP_FAILURE; } #ifdef HAVE_FILENO_UNLOCKED ret = dup2(opts->p_stdout[1], fileno_unlocked(stdout)); #else ret = dup2(opts->p_stdout[1], fileno(stdout)); #endif if (ret < 0) { return PMIX_ERR_PIPE_SETUP_FAILURE; } close(opts->p_stdout[1]); } else { #ifdef HAVE_FILENO_UNLOCKED if (opts->p_stdout[1] != fileno_unlocked(stdout)) { ret = dup2(opts->p_stdout[1], fileno_unlocked(stdout)); #else if (opts->p_stdout[1] != fileno(stdout)) { ret = dup2(opts->p_stdout[1], fileno(stdout)); #endif if (ret < 0) { return PMIX_ERR_PIPE_SETUP_FAILURE; } close(opts->p_stdout[1]); } } if (opts->connect_stdin) { #ifdef HAVE_FILENO_UNLOCKED if (opts->p_stdin[0] != fileno_unlocked(stdin)) { ret = dup2(opts->p_stdin[0], fileno_unlocked(stdin)); #else if (opts->p_stdin[0] != fileno(stdin)) { ret = dup2(opts->p_stdin[0], fileno(stdin)); #endif if (ret < 0) { return PMIX_ERR_PIPE_SETUP_FAILURE; } close(opts->p_stdin[0]); } } else { int fd; /* connect input to /dev/null */ fd = open("/dev/null", O_RDONLY, 0); if (0 > fd) { pmix_output(0, "Failed system call to open on /dev/null - errno: %s(%d)", strerror(errno), errno); return PRTE_ERROR; } if (fd != fileno(stdin)) { dup2(fd, fileno(stdin)); } close(fd); } #ifdef HAVE_FILENO_UNLOCKED if (opts->p_stderr[1] != fileno_unlocked(stderr)) { ret = dup2(opts->p_stderr[1], fileno_unlocked(stderr)); #else if (opts->p_stderr[1] != fileno(stderr)) { ret = dup2(opts->p_stderr[1], fileno(stderr)); #endif if (ret < 0) { return PMIX_ERR_PIPE_SETUP_FAILURE; } close(opts->p_stderr[1]); } return PRTE_SUCCESS; } int prte_iof_base_setup_parent(const pmix_proc_t *name, prte_iof_base_io_conf_t *opts) { int ret; /* connect stdin endpoint */ if (opts->connect_stdin) { /* and connect the pty to stdin */ ret = prte_iof.pull(name, PRTE_IOF_STDIN, opts->p_stdin[1]); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); return ret; } } /* connect read ends to IOF */ ret = prte_iof.push(name, PRTE_IOF_STDOUT, opts->p_stdout[0]); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); return ret; } ret = prte_iof.push(name, PRTE_IOF_STDERR, opts->p_stderr[0]); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); return ret; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/iof/base/owner.txt0000664000175000017500000000024415203542132020550 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/iof/base/iof_base_frame.c0000664000175000017500000002034315203542132021744 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. * Copyright (c) 2013-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" /* * The following file was created by configure. It contains extern * statements and the definition of an array of pointers to each * component's public prte_base_component_t struct. */ #include "src/mca/iof/base/static-components.h" prte_iof_base_module_t prte_iof = {0}; /* * Global variables */ int prte_iof_base_output_limit = 0; static int prte_iof_base_register(pmix_mca_base_register_flag_t flags) { PRTE_HIDE_UNUSED_PARAMS(flags); /* check for maximum number of pending output messages */ prte_iof_base_output_limit = (size_t) INT_MAX; (void) pmix_mca_base_var_register("prte", "iof", "base", "output_limit", "Maximum backlog of output messages [default: unlimited]", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_iof_base_output_limit); return PRTE_SUCCESS; } static int prte_iof_base_close(void) { /* Close the selected component */ if (NULL != prte_iof.finalize) { prte_iof.finalize(); } return pmix_mca_base_framework_components_close(&prte_iof_base_framework, NULL); } /** * Function for finding and opening either all MCA components, or the one * that was specifically requested via a MCA parameter. */ static int prte_iof_base_open(pmix_mca_base_open_flag_t flags) { /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_iof_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, iof, "PRTE I/O Forwarding", prte_iof_base_register, prte_iof_base_open, prte_iof_base_close, prte_iof_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); static void lkcbfunc(pmix_status_t status, void *cbdata) { prte_iof_deliver_t *p = (prte_iof_deliver_t*)cbdata; /* nothing to do here - we use this solely to * ensure that IOF_deliver doesn't block */ if (PMIX_SUCCESS != status) { PMIX_ERROR_LOG(status); } PMIX_RELEASE(p); } void prte_iof_base_output(const pmix_proc_t *source, pmix_iof_channel_t channel, char *string) { prte_iof_deliver_t *p; pmix_status_t rc; p = PMIX_NEW(prte_iof_deliver_t); PMIX_XFER_PROCID(&p->source, source); p->bo.bytes = string; p->bo.size = strlen(string); rc = PMIx_server_IOF_deliver(&p->source, channel, &p->bo, NULL, 0, lkcbfunc, (void*)p); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(p); // releases string } } /* class instances */ static void prte_iof_base_proc_construct(prte_iof_proc_t *ptr) { ptr->stdinev = NULL; ptr->revstdout = NULL; ptr->revstderr = NULL; } static void prte_iof_base_proc_destruct(prte_iof_proc_t *ptr) { if (NULL != ptr->stdinev) { PMIX_RELEASE(ptr->stdinev); } if (NULL != ptr->revstdout) { PMIX_RELEASE(ptr->revstdout); } if (NULL != ptr->revstderr) { PMIX_RELEASE(ptr->revstderr); } } PMIX_CLASS_INSTANCE(prte_iof_proc_t, pmix_list_item_t, prte_iof_base_proc_construct, prte_iof_base_proc_destruct); static void prte_iof_base_sink_construct(prte_iof_sink_t *ptr) { PMIX_LOAD_PROCID(&ptr->daemon, NULL, PMIX_RANK_INVALID); ptr->wev = PMIX_NEW(prte_iof_write_event_t); ptr->xoff = false; ptr->exclusive = false; ptr->closed = false; } static void prte_iof_base_sink_destruct(prte_iof_sink_t *ptr) { if (NULL != ptr->wev) { PMIX_OUTPUT_VERBOSE((20, prte_iof_base_framework.framework_output, "%s iof: closing sink for process %s on fd %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&ptr->name), ptr->wev->fd)); PMIX_RELEASE(ptr->wev); } } PMIX_CLASS_INSTANCE(prte_iof_sink_t, pmix_list_item_t, prte_iof_base_sink_construct, prte_iof_base_sink_destruct); static void prte_iof_base_read_event_construct(prte_iof_read_event_t *rev) { rev->proc = NULL; rev->fd = -1; rev->active = false; rev->activated = false; rev->always_readable = false; rev->ev = prte_event_alloc(); rev->sink = NULL; rev->tv.tv_sec = 0; rev->tv.tv_usec = 0; } static void prte_iof_base_read_event_destruct(prte_iof_read_event_t *rev) { prte_iof_proc_t *proct = (prte_iof_proc_t *) rev->proc; if (0 <= rev->fd) { prte_event_free(rev->ev); PMIX_OUTPUT_VERBOSE((20, prte_iof_base_framework.framework_output, "%s iof: closing fd %d for process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), rev->fd, (NULL == proct) ? "UNKNOWN" : PRTE_NAME_PRINT(&proct->name))); close(rev->fd); rev->fd = -1; } else { free(rev->ev); } if (NULL != rev->sink) { PMIX_RELEASE(rev->sink); } if (NULL != proct) { PMIX_RELEASE(proct); } } PMIX_CLASS_INSTANCE(prte_iof_read_event_t, pmix_object_t, prte_iof_base_read_event_construct, prte_iof_base_read_event_destruct); static void prte_iof_base_write_event_construct(prte_iof_write_event_t *wev) { wev->pending = false; wev->always_writable = false; wev->fd = -1; PMIX_CONSTRUCT(&wev->outputs, pmix_list_t); wev->ev = prte_event_alloc(); wev->tv.tv_sec = 0; wev->tv.tv_usec = 0; } static void prte_iof_base_write_event_destruct(prte_iof_write_event_t *wev) { if (0 <= wev->fd) { prte_event_free(wev->ev); } else { free(wev->ev); } if (2 < wev->fd) { PMIX_OUTPUT_VERBOSE((20, prte_iof_base_framework.framework_output, "%s iof: closing fd %d for write event", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); close(wev->fd); } PMIX_DESTRUCT(&wev->outputs); } PMIX_CLASS_INSTANCE(prte_iof_write_event_t, pmix_list_item_t, prte_iof_base_write_event_construct, prte_iof_base_write_event_destruct); PMIX_CLASS_INSTANCE(prte_iof_write_output_t, pmix_list_item_t, NULL, NULL); static void pdcon(prte_iof_deliver_t *p) { p->bo.bytes = NULL; p->bo.size = 0; } static void pddes(prte_iof_deliver_t *p) { if (NULL != p->bo.bytes) { free(p->bo.bytes); } } PMIX_CLASS_INSTANCE(prte_iof_deliver_t, pmix_object_t, pdcon, pddes); prrte-4.1.1~rc2/src/mca/iof/base/iof_base_output.c0000664000175000017500000001526015203542132022214 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include #include #ifdef HAVE_UNISTD_H # include #endif #include #include #include "src/util/pmix_output.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" int prte_iof_base_write_output(const pmix_proc_t *name, prte_iof_tag_t stream, const unsigned char *data, int numbytes, prte_iof_write_event_t *channel) { prte_iof_write_output_t *output; int num_buffered; PRTE_HIDE_UNUSED_PARAMS(stream); PMIX_OUTPUT_VERBOSE( (1, prte_iof_base_framework.framework_output, "%s write:output setting up to write %d bytes to stdin for %s on fd %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes, PRTE_NAME_PRINT(name), (NULL == channel) ? -1 : channel->fd)); if (NULL == channel) { return 0; } /* setup output object */ output = PMIX_NEW(prte_iof_write_output_t); /* copy over the data to be written */ if (0 < numbytes) { /* don't copy 0 bytes - we just need to pass * the zero bytes so the fd can be closed * after it writes everything out */ memcpy(output->data, data, numbytes); } output->numbytes = numbytes; /* add this data to the write list for this fd */ pmix_list_append(&channel->outputs, &output->super); /* record how big the buffer is */ num_buffered = pmix_list_get_size(&channel->outputs); /* is the write event issued? */ if (!channel->pending) { /* issue it */ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s write:output adding write event", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_IOF_SINK_ACTIVATE(channel); } return num_buffered; } void prte_iof_base_write_handler(int _fd, short event, void *cbdata) { prte_iof_sink_t *sink = (prte_iof_sink_t *) cbdata; prte_iof_write_event_t *wev = sink->wev; pmix_list_item_t *item; prte_iof_write_output_t *output; int num_written, total_written = 0; PRTE_HIDE_UNUSED_PARAMS(_fd, event); PMIX_ACQUIRE_OBJECT(sink); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s write:handler writing data to %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); while (NULL != (item = pmix_list_remove_first(&wev->outputs))) { output = (prte_iof_write_output_t *) item; if (0 == output->numbytes) { /* indicates we are to close this stream */ PMIX_RELEASE(sink); return; } num_written = write(wev->fd, output->data, output->numbytes); if (num_written < 0) { if (EAGAIN == errno || EINTR == errno) { /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* if the list is getting too large, abort */ if (prte_iof_base_output_limit < (int)pmix_list_get_size(&wev->outputs)) { pmix_output(0, "IO Forwarding is running too far behind - something is " "blocking us from writing"); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); goto ABORT; } /* leave the write event running so it will call us again * when the fd is ready. */ goto NEXT_CALL; } /* otherwise, something bad happened so all we can do is abort * this attempt */ PMIX_RELEASE(output); goto ABORT; } else if (num_written < output->numbytes) { /* incomplete write - adjust data to avoid duplicate output */ memmove(output->data, &output->data[num_written], output->numbytes - num_written); /* adjust the number of bytes remaining to be written */ output->numbytes -= num_written; /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* if the list is getting too large, abort */ if (prte_iof_base_output_limit < (int)pmix_list_get_size(&wev->outputs)) { pmix_output(0, "IO Forwarding is running too far behind - something is blocking us " "from writing"); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); goto ABORT; } /* leave the write event running so it will call us again * when the fd is ready */ goto NEXT_CALL; } PMIX_RELEASE(output); total_written += num_written; if (wev->always_writable && (PRTE_IOF_SINK_BLOCKSIZE <= total_written)) { /* If this is a regular file it will never tell us it will block * Write no more than PRTE_IOF_REGULARF_BLOCK at a time allowing * other fds to progress */ goto NEXT_CALL; } } ABORT: wev->pending = false; PMIX_POST_OBJECT(wev); return; NEXT_CALL: PRTE_IOF_SINK_ACTIVATE(wev); } prrte-4.1.1~rc2/src/mca/iof/base/base.h0000664000175000017500000002544315203542132017750 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * I/O Forwarding Service */ #ifndef MCA_IOF_BASE_H #define MCA_IOF_BASE_H #include "prte_config.h" #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_UIO_H # include #endif #ifdef HAVE_NET_UIO_H # include #endif #ifdef HAVE_UNISTD_H # include #endif #include #include "src/class/pmix_bitmap.h" #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_fd.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/iof/iof.h" #include "src/rml/rml_types.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" BEGIN_C_DECLS /* * MCA framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_iof_base_framework; /* * Select an available component. */ PRTE_EXPORT int prte_iof_base_select(void); /* * Maximum size of single msg */ #define PRTE_IOF_BASE_MSG_MAX 4096 #define PRTE_IOF_BASE_TAG_MAX 1024 #define PRTE_IOF_BASE_TAGGED_OUT_MAX 8192 #define PRTE_IOF_MAX_INPUT_BUFFERS 50 typedef struct { pmix_list_item_t super; bool pending; bool always_writable; prte_event_t *ev; struct timeval tv; int fd; pmix_list_t outputs; } prte_iof_write_event_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_write_event_t); typedef struct { pmix_list_item_t super; pmix_proc_t name; pmix_proc_t daemon; prte_iof_tag_t tag; prte_iof_write_event_t *wev; bool xoff; bool exclusive; bool closed; } prte_iof_sink_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_sink_t); struct prte_iof_proc_t; typedef struct { pmix_object_t super; struct prte_iof_proc_t *proc; prte_event_t *ev; struct timeval tv; int fd; prte_iof_tag_t tag; bool active; bool activated; bool always_readable; prte_iof_sink_t *sink; } prte_iof_read_event_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_read_event_t); typedef struct { pmix_list_item_t super; pmix_proc_t name; prte_iof_sink_t *stdinev; prte_iof_read_event_t *revstdout; prte_iof_read_event_t *revstderr; } prte_iof_proc_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_proc_t); typedef struct { pmix_list_item_t super; char data[PRTE_IOF_BASE_TAGGED_OUT_MAX]; int numbytes; } prte_iof_write_output_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_write_output_t); typedef struct{ pmix_object_t super; pmix_proc_t source; pmix_byte_object_t bo; } prte_iof_deliver_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_iof_deliver_t); /* Write event macro's */ static inline bool prte_iof_base_fd_always_ready(int fd) { return pmix_fd_is_regular(fd) || (pmix_fd_is_chardev(fd) && !isatty(fd)) || pmix_fd_is_blkdev(fd); } #define PRTE_IOF_SINK_BLOCKSIZE (1024) #define PRTE_IOF_SINK_ACTIVATE(wev) \ do { \ struct timeval *tv = NULL; \ wev->pending = true; \ PMIX_POST_OBJECT(wev); \ if (wev->always_writable) { \ /* Regular is always write ready. Use timer to activate */ \ tv = &wev->tv; \ } \ if (prte_event_add(wev->ev, tv)) { \ PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); \ } \ } while (0); /* define an output "sink", adding it to the provided * endpoint list for this proc */ #define PRTE_IOF_SINK_DEFINE(snk, nm, fid, tg, wrthndlr) \ do { \ prte_iof_sink_t *ep; \ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, \ "defining endpt: file %s line %d fd %d", __FILE__, __LINE__, (fid))); \ ep = PMIX_NEW(prte_iof_sink_t); \ PMIX_LOAD_PROCID(&ep->name, (nm)->nspace, (nm)->rank); \ ep->tag = (tg); \ if (0 <= (fid)) { \ ep->wev->fd = (fid); \ ep->wev->always_writable = prte_iof_base_fd_always_ready(fid); \ if (ep->wev->always_writable) { \ prte_event_evtimer_set(prte_event_base, ep->wev->ev, wrthndlr, ep); \ } else { \ prte_event_set(prte_event_base, ep->wev->ev, ep->wev->fd, PRTE_EV_WRITE, wrthndlr, \ ep); \ } \ } \ *(snk) = ep; \ PMIX_POST_OBJECT(ep); \ } while (0); /* Read event macro's */ #define PRTE_IOF_READ_ADDEV(rev) \ do { \ struct timeval *tv = NULL; \ if (rev->always_readable) { \ tv = &rev->tv; \ } \ if (prte_event_add(rev->ev, tv)) { \ PRTE_ERROR_LOG(PRTE_ERR_BAD_PARAM); \ } \ } while (0); #define PRTE_IOF_READ_ACTIVATE(rev) \ do { \ rev->active = true; \ PMIX_POST_OBJECT(rev); \ PRTE_IOF_READ_ADDEV(rev); \ } while (0); /* add list of structs that has name of proc + prte_iof_tag_t - when * defining a read event, search list for proc, add flag to the tag. * when closing a read fd, find proc on list and zero out that flag * when all flags = 0, then iof is complete - set message event to * daemon processor indicating proc iof is terminated */ #define PRTE_IOF_READ_EVENT(rv, p, fid, tg, cbfunc, actv) \ do { \ prte_iof_read_event_t *rev; \ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, \ "%s defining read event for %s: %s %d", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&(p)->name), \ __FILE__, __LINE__)); \ rev = PMIX_NEW(prte_iof_read_event_t); \ PMIX_RETAIN((p)); \ rev->proc = (struct prte_iof_proc_t *) (p); \ rev->tag = (tg); \ rev->fd = (fid); \ rev->always_readable = prte_iof_base_fd_always_ready(fid); \ *(rv) = rev; \ if (rev->always_readable) { \ prte_event_evtimer_set(prte_event_base, rev->ev, (cbfunc), rev); \ } else { \ prte_event_set(prte_event_base, rev->ev, (fid), PRTE_EV_READ, (cbfunc), rev); \ } \ if ((actv)) { \ PRTE_IOF_READ_ACTIVATE(rev) \ } \ } while (0); PRTE_EXPORT int prte_iof_base_flush(void); PRTE_EXPORT extern int prte_iof_base_output_limit; /* base functions */ PRTE_EXPORT int prte_iof_base_write_output(const pmix_proc_t *name, prte_iof_tag_t stream, const unsigned char *data, int numbytes, prte_iof_write_event_t *channel); PRTE_EXPORT void prte_iof_base_write_handler(int fd, short event, void *cbdata); PRTE_EXPORT void prte_iof_base_output(const pmix_proc_t *source, pmix_iof_channel_t channel, char *string); END_C_DECLS #endif /* MCA_IOF_BASE_H */ prrte-4.1.1~rc2/src/mca/iof/base/Makefile.am0000664000175000017500000000224215203542132020711 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # headers += \ base/base.h \ base/iof_base_setup.h libprtemca_iof_la_SOURCES += \ base/iof_base_frame.c \ base/iof_base_select.c \ base/iof_base_output.c \ base/iof_base_setup.c prrte-4.1.1~rc2/src/mca/iof/base/iof_base_select.c0000664000175000017500000000450215203542132022130 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2007 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/errmgr/errmgr.h" #include "src/util/proc_info.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" /** * Call the query function on all available components to find out if * they want to run. Select the single component with the highest * priority. */ int prte_iof_base_select(void) { prte_iof_base_component_t *best_component = NULL; prte_iof_base_module_t *best_module = NULL; int rc; /* * Select the best component */ if (PRTE_SUCCESS != pmix_mca_base_select("iof", prte_iof_base_framework.framework_output, &prte_iof_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL)) { /* this is a problem */ return PRTE_ERR_NOT_FOUND; } /* Save the winner */ prte_iof = *best_module; /* init it */ if (NULL != prte_iof.init) { if (PRTE_SUCCESS != (rc = prte_iof.init())) { PRTE_ERROR_LOG(rc); return rc; } } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/iof/Makefile.am0000664000175000017500000000244015203542132017777 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AM_CPPFLAGS = $(LTDLINCL) EXTRA_DIST = README.txt # main library setup noinst_LTLIBRARIES = libprtemca_iof.la libprtemca_iof_la_SOURCES = # local files headers = iof.h iof_types.h libprtemca_iof_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/iof/prted/0000775000175000017500000000000015203542132017061 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/iof/prted/iof_prted_component.c0000664000175000017500000000563115203542132023267 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/mca/base/pmix_base.h" #include "src/util/proc_info.h" #include "iof_prted.h" /* * Local functions */ static int prte_iof_prted_open(void); static int prte_iof_prted_close(void); static int prte_iof_prted_query(pmix_mca_base_module_t **module, int *priority); /* * Public string showing the iof prted component version number */ const char *prte_mca_iof_prted_component_version_string = "PRTE prted iof MCA component version " PRTE_VERSION; prte_mca_iof_prted_component_t prte_mca_iof_prted_component = { .super = { PRTE_IOF_BASE_VERSION_2_0_0, .pmix_mca_component_name = "prted", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open, close, and query functions */ .pmix_mca_open_component = prte_iof_prted_open, .pmix_mca_close_component = prte_iof_prted_close, .pmix_mca_query_component = prte_iof_prted_query, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, iof, prted) /** * component open/close/init function */ static int prte_iof_prted_open(void) { /* Nothing to do */ return PRTE_SUCCESS; } static int prte_iof_prted_close(void) { return PRTE_SUCCESS; } static int prte_iof_prted_query(pmix_mca_base_module_t **module, int *priority) { /* if we are not a daemon, then don't use this module */ if (!PRTE_PROC_IS_DAEMON) { *module = NULL; *priority = -1; return PRTE_ERROR; } *priority = 80; *module = (pmix_mca_base_module_t *) &prte_iof_prted_module; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/iof/prted/owner.txt0000664000175000017500000000024415203542132020754 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/iof/prted/iof_prted.h0000664000175000017500000000611715203542132021212 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * The prted IOF component is used in daemons. It is used * to prted all IOF actions back to the "hnp" IOF component (i.e., the * IOF component that runs in the HNP). The prted IOF component is * loaded in an prted and then tied to the stdin, stdout, * and stderr streams of created child processes via pipes. The prted * IOF component in the prted then acts as the relay between the * stdin/stdout/stderr pipes and the IOF component in the HNP. * This design allows us to manipulate stdin/stdout/stderr from before * main() in the child process. * * Much of the intelligence of this component is actually contained in * iof_base_endpoint.c (reading and writing to local file descriptors, * setting up events based on file descriptors, etc.). * * A non-blocking OOB receive is posted at the initialization of this * component to receive all messages from the HNP (e.g., data * fragments from streams, ACKs to fragments). * * Flow control is employed on a per-stream basis to ensure that * SOURCEs don't overwhelm SINK resources (E.g., send an entire input * file to an prted before the target process has read any of it). * */ #ifndef PRTE_IOF_PRTED_H #define PRTE_IOF_PRTED_H #include "prte_config.h" #include "src/class/pmix_list.h" #include "src/mca/iof/iof.h" #include "src/rml/rml_types.h" BEGIN_C_DECLS /** * IOF PRTED Component */ struct prte_mca_iof_prted_component_t { prte_iof_base_component_t super; pmix_list_t procs; bool xoff; }; typedef struct prte_mca_iof_prted_component_t prte_mca_iof_prted_component_t; PRTE_MODULE_EXPORT extern prte_mca_iof_prted_component_t prte_mca_iof_prted_component; extern prte_iof_base_module_t prte_iof_prted_module; void prte_iof_prted_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); void prte_iof_prted_read_handler(int fd, short event, void *data); void prte_iof_prted_send_xonxoff(prte_iof_tag_t tag); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/iof/prted/iof_prted.c0000664000175000017500000003464515203542132021214 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 Mellanox Technologies. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/util/pmix_output.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #ifdef HAVE_FCNTL_H # include #else # ifdef HAVE_SYS_FCNTL_H # include # endif #endif #include "src/pmix/pmix-internal.h" #include "src/util/pmix_os_dirpath.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/odls/odls_types.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/base/iof_base_setup.h" #include "src/mca/iof/iof.h" #include "iof_prted.h" /* LOCAL FUNCTIONS */ static void stdin_write_handler(int fd, short event, void *cbdata); /* API FUNCTIONS */ static int init(void); static int prted_push(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd); static int prted_pull(const pmix_proc_t *src_name, prte_iof_tag_t src_tag, int fd); static int prted_close(const pmix_proc_t *peer, prte_iof_tag_t source_tag); static void prted_complete(const prte_job_t *jdata); static int finalize(void); /* The API's in this module are solely used to support LOCAL * procs - i.e., procs that are co-located to the daemon. Output * from local procs is automatically sent to the HNP for output * and possible forwarding to other requestors. */ prte_iof_base_module_t prte_iof_prted_module = { .init = init, .push = prted_push, .pull = prted_pull, .close = prted_close, .complete = prted_complete, .finalize = finalize, }; static int init(void) { /* post a non-blocking RML receive to get messages from the HNP IOF component */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_IOF_PROXY, PRTE_RML_PERSISTENT, prte_iof_prted_recv, NULL); /* setup the local global variables */ PMIX_CONSTRUCT(&prte_mca_iof_prted_component.procs, pmix_list_t); prte_mca_iof_prted_component.xoff = false; return PRTE_SUCCESS; } /** * Push data from the specified file descriptor * to the HNP */ static int prted_push(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd) { int flags; prte_iof_proc_t *proct; prte_job_t *jobdat = NULL; PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:prted pushing fd %d for process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), fd, PRTE_NAME_PRINT(dst_name))); /* set the file descriptor to non-blocking - do this before we setup * and activate the read event in case it fires right away */ if ((flags = fcntl(fd, F_GETFL, 0)) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_GETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } else { flags |= O_NONBLOCK; if (fcntl(fd, F_SETFL, flags) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_SETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } } /* do we already have this process in our list? */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_prted_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, dst_name)) { /* found it */ goto SETUP; } } /* if we get here, then we don't yet have this proc in our list */ proct = PMIX_NEW(prte_iof_proc_t); PMIX_XFER_PROCID(&proct->name, dst_name); pmix_list_append(&prte_mca_iof_prted_component.procs, &proct->super); SETUP: /* get the local jobdata for this proc */ if (NULL == (jobdat = prte_get_job_data_object(proct->name.nspace))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } /* define a read event and activate it */ if (src_tag & PRTE_IOF_STDOUT) { PRTE_IOF_READ_EVENT(&proct->revstdout, proct, fd, PRTE_IOF_STDOUT, prte_iof_prted_read_handler, false); } else if (src_tag & PRTE_IOF_STDERR) { PRTE_IOF_READ_EVENT(&proct->revstderr, proct, fd, PRTE_IOF_STDERR, prte_iof_prted_read_handler, false); } /* if -all- of the readevents for this proc have been defined, then * activate them. Otherwise, we can think that the proc is complete * because one of the readevents fires -prior- to all of them having * been defined! */ if (NULL != proct->revstdout && NULL != proct->revstderr) { if (!proct->revstdout->activated) { PRTE_IOF_READ_ACTIVATE(proct->revstdout); proct->revstdout->activated = true; } if (!proct->revstderr->activated) { PRTE_IOF_READ_ACTIVATE(proct->revstderr); proct->revstderr->activated = true; } } return PRTE_SUCCESS; } /** * Pull for a daemon tells * us that any info we receive from the HNP that is targeted * for stdin of the specified process should be fed down the * indicated file descriptor. Thus, all we need to do here * is define a local endpoint so we know where to feed anything * that comes to us */ static int prted_pull(const pmix_proc_t *dst_name, prte_iof_tag_t src_tag, int fd) { prte_iof_proc_t *proct; prte_ns_cmp_bitmask_t mask = PRTE_NS_CMP_ALL; int flags; /* this is a local call - only stdin is suppprted */ if (PRTE_IOF_STDIN != src_tag) { return PRTE_ERR_NOT_SUPPORTED; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:prted pulling fd %d for process %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), fd, PRTE_NAME_PRINT(dst_name))); /* set the file descriptor to non-blocking - do this before we setup * the sink in case it fires right away */ if ((flags = fcntl(fd, F_GETFL, 0)) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_GETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } else { flags |= O_NONBLOCK; if (fcntl(fd, F_SETFL, flags) < 0) { pmix_output(prte_iof_base_framework.framework_output, "[%s:%d]: fcntl(F_SETFL) failed with errno=%d\n", __FILE__, __LINE__, errno); } } /* do we already have this process in our list? */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_prted_component.procs, prte_iof_proc_t) { if (PRTE_EQUAL == prte_util_compare_name_fields(mask, &proct->name, dst_name)) { /* found it */ goto SETUP; } } /* if we get here, then we don't yet have this proc in our list */ proct = PMIX_NEW(prte_iof_proc_t); PMIX_XFER_PROCID(&proct->name, dst_name); pmix_list_append(&prte_mca_iof_prted_component.procs, &proct->super); SETUP: PRTE_IOF_SINK_DEFINE(&proct->stdinev, dst_name, fd, PRTE_IOF_STDIN, stdin_write_handler); return PRTE_SUCCESS; } /* * One of our local procs wants us to close the specifed * stream(s), thus terminating any potential io to/from it. * For the prted, this just means closing the local fd */ static int prted_close(const pmix_proc_t *peer, prte_iof_tag_t source_tag) { prte_iof_proc_t *proct; PMIX_LIST_FOREACH(proct, &prte_mca_iof_prted_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_PROCID(&proct->name, peer)) { if (PRTE_IOF_STDIN & source_tag) { if (NULL != proct->stdinev) { PMIX_RELEASE(proct->stdinev); } proct->stdinev = NULL; } if ((PRTE_IOF_STDOUT & source_tag) || (PRTE_IOF_STDMERGE & source_tag)) { if (NULL != proct->revstdout) { PMIX_RELEASE(proct->revstdout); } proct->revstdout = NULL; } if (PRTE_IOF_STDERR & source_tag) { if (NULL != proct->revstderr) { PMIX_RELEASE(proct->revstderr); } proct->revstderr = NULL; } /* if we closed them all, then remove this proc */ if (NULL == proct->stdinev && NULL == proct->revstdout && NULL == proct->revstderr) { pmix_list_remove_item(&prte_mca_iof_prted_component.procs, &proct->super); PMIX_RELEASE(proct); } break; } } return PRTE_SUCCESS; } static void prted_complete(const prte_job_t *jdata) { prte_iof_proc_t *proct, *next; /* cleanout any lingering sinks */ PMIX_LIST_FOREACH_SAFE(proct, next, &prte_mca_iof_prted_component.procs, prte_iof_proc_t) { if (PMIX_CHECK_NSPACE(jdata->nspace, proct->name.nspace)) { pmix_list_remove_item(&prte_mca_iof_prted_component.procs, &proct->super); PMIX_RELEASE(proct); } } } static int finalize(void) { PMIX_LIST_DESTRUCT(&prte_mca_iof_prted_component.procs); /* Cancel the RML receive */ PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_IOF_PROXY); return PRTE_SUCCESS; } static void stdin_write_handler(int _fd, short event, void *cbdata) { prte_iof_sink_t *sink = (prte_iof_sink_t *) cbdata; prte_iof_write_event_t *wev = sink->wev; pmix_list_item_t *item; prte_iof_write_output_t *output; int num_written; PRTE_HIDE_UNUSED_PARAMS(_fd, event); PMIX_ACQUIRE_OBJECT(sink); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s prted:stdin:write:handler writing data to %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); wev->pending = false; while (NULL != (item = pmix_list_remove_first(&wev->outputs))) { output = (prte_iof_write_output_t *) item; if (0 == output->numbytes) { /* this indicates we are to close the fd - there is * nothing to write */ PMIX_OUTPUT_VERBOSE( (20, prte_iof_base_framework.framework_output, "%s iof:prted closing fd %d on write event due to zero bytes output", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); PMIX_RELEASE(wev); sink->wev = NULL; return; } num_written = write(wev->fd, output->data, output->numbytes); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s prted:stdin:write:handler wrote %d bytes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), num_written)); if (num_written < 0) { if (EAGAIN == errno || EINTR == errno) { /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* leave the write event running so it will call us again * when the fd is ready. */ PRTE_IOF_SINK_ACTIVATE(wev); goto CHECK; } /* otherwise, something bad happened so all we can do is declare an * error and abort */ PMIX_RELEASE(output); PMIX_OUTPUT_VERBOSE( (20, prte_iof_base_framework.framework_output, "%s iof:prted closing fd %d on write event due to negative bytes written", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), wev->fd)); PMIX_RELEASE(wev); sink->wev = NULL; /* tell the HNP to stop sending us stuff */ if (!prte_mca_iof_prted_component.xoff) { prte_mca_iof_prted_component.xoff = true; prte_iof_prted_send_xonxoff(PRTE_IOF_XOFF); } return; } else if (num_written < output->numbytes) { PMIX_OUTPUT_VERBOSE( (1, prte_iof_base_framework.framework_output, "%s prted:stdin:write:handler incomplete write %d - adjusting data", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), num_written)); /* incomplete write - adjust data to avoid duplicate output */ memmove(output->data, &output->data[num_written], output->numbytes - num_written); /* push this item back on the front of the list */ pmix_list_prepend(&wev->outputs, item); /* leave the write event running so it will call us again * when the fd is ready. */ PRTE_IOF_SINK_ACTIVATE(wev); goto CHECK; } PMIX_RELEASE(output); } CHECK: if (prte_mca_iof_prted_component.xoff) { /* if we have told the HNP to stop reading stdin, see if * the proc has absorbed enough to justify restart * * RHC: Note that when multiple procs want stdin, we * can get into a fight between a proc turnin stdin * back "on" and other procs turning it "off". There * is no clear way to resolve this as different procs * may take input at different rates. */ if (pmix_list_get_size(&wev->outputs) < PRTE_IOF_MAX_INPUT_BUFFERS) { /* restart the read */ prte_mca_iof_prted_component.xoff = false; prte_iof_prted_send_xonxoff(PRTE_IOF_XON); } } } prrte-4.1.1~rc2/src/mca/iof/prted/iof_prted_receive.c0000664000175000017500000001325715203542132022712 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof_types.h" #include "iof_prted.h" void prte_iof_prted_send_xonxoff(prte_iof_tag_t tag) { pmix_data_buffer_t *buf; int rc; PMIX_DATA_BUFFER_CREATE(buf); /* pack the tag - we do this first so that flow control messages can * consist solely of the tag */ rc = PMIx_Data_pack(NULL, buf, &tag, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return; } PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s sending %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (PRTE_IOF_XON == tag) ? "xon" : "xoff")); /* send the buffer to the HNP */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, buf, PRTE_RML_TAG_IOF_HNP); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); } } /* * The only messages coming to an prted are either: * * (a) stdin, which is to be copied to whichever local * procs "pull'd" a copy * * (b) flow control messages */ void prte_iof_prted_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { unsigned char data[PRTE_IOF_BASE_MSG_MAX]; prte_iof_tag_t stream; int32_t count, numbytes; pmix_proc_t target; prte_iof_proc_t *proct; int rc; PRTE_HIDE_UNUSED_PARAMS(status, sender, tag, cbdata); /* see what stream generated this data */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &stream, &count, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } /* if this isn't stdin, then we have an error */ if (PRTE_IOF_STDIN != stream) { PRTE_ERROR_LOG(PRTE_ERR_COMM_FAILURE); return; } /* unpack the intended target */ count = 1; rc = PMIx_Data_unpack(NULL, buffer, &target, &count, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } /* unpack the data */ numbytes = PRTE_IOF_BASE_MSG_MAX; rc = PMIx_Data_unpack(NULL, buffer, data, &numbytes, PMIX_BYTE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } /* numbytes will contain the actual #bytes that were sent */ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s unpacked %d bytes for local proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes, PRTE_NAME_PRINT(&target))); /* cycle through our list of procs */ PMIX_LIST_FOREACH(proct, &prte_mca_iof_prted_component.procs, prte_iof_proc_t) { /* is this intended for this jobid? */ if (PMIX_CHECK_NSPACE(target.nspace, proct->name.nspace)) { /* yes - is this intended for all vpids or this vpid? */ if (PMIX_CHECK_RANK(target.rank, proct->name.rank)) { PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s writing data to local proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proct->name))); if (NULL == proct->stdinev) { continue; } /* send the bytes down the pipe - we even send 0 byte events * down the pipe so it forces out any preceding data before * closing the output stream */ if (PRTE_IOF_MAX_INPUT_BUFFERS < prte_iof_base_write_output(&target, stream, data, numbytes, proct->stdinev->wev)) { /* getting too backed up - tell the HNP to hold off any more input if we * haven't already told it */ if (!prte_mca_iof_prted_component.xoff) { prte_mca_iof_prted_component.xoff = true; prte_iof_prted_send_xonxoff(PRTE_IOF_XOFF); } } } } } } prrte-4.1.1~rc2/src/mca/iof/prted/iof_prted_read.c0000664000175000017500000001460615203542132022202 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/odls/odls_types.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/mca/iof/base/base.h" #include "src/mca/iof/iof.h" #include "iof_prted.h" static void lkcbfunc(pmix_status_t status, void *cbdata) { prte_iof_deliver_t *p = (prte_iof_deliver_t*)cbdata; /* nothing to do here - we use this solely to * ensure that IOF_deliver doesn't block */ if (PMIX_SUCCESS != status) { PMIX_ERROR_LOG(status); } PMIX_RELEASE(p); } void prte_iof_prted_read_handler(int fd, short event, void *cbdata) { prte_iof_read_event_t *rev = (prte_iof_read_event_t *) cbdata; unsigned char data[PRTE_IOF_BASE_MSG_MAX]; pmix_data_buffer_t *buf = NULL; int rc; int32_t numbytes; prte_iof_proc_t *proct = (prte_iof_proc_t *) rev->proc; prte_iof_deliver_t *p; pmix_iof_channel_t pchan; pmix_status_t prc; PRTE_HIDE_UNUSED_PARAMS(event); PMIX_ACQUIRE_OBJECT(rev); /* As we may use timer events, fd can be bogus (-1) * use the right one here */ fd = rev->fd; /* read up to the fragment size */ numbytes = read(fd, data, sizeof(data)); PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s read %d bytes from %s of %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes, (PRTE_IOF_STDOUT & rev->tag) ? "stdout" : ((PRTE_IOF_STDERR & rev->tag) ? "stderr" : "stddiag"), PRTE_NAME_PRINT(&proct->name))); if (NULL == proct) { /* nothing we can do */ PRTE_ERROR_LOG(PRTE_ERR_ADDRESSEE_UNKNOWN); return; } if (numbytes <= 0) { if (0 > numbytes) { /* either we have a connection error or it was a non-blocking read */ if (EAGAIN == errno || EINTR == errno) { /* non-blocking, retry */ PRTE_IOF_READ_ACTIVATE(rev); return; } } /* go down and close the fd etc */ goto CLEAN_RETURN; } /* give the PMIx lib a chance to output it if requested */ pchan = 0; if (PRTE_IOF_STDOUT & rev->tag) { pchan |= PMIX_FWD_STDOUT_CHANNEL; } if (PRTE_IOF_STDERR & rev->tag) { pchan |= PMIX_FWD_STDERR_CHANNEL; } if (PRTE_IOF_STDDIAG & rev->tag) { pchan |= PMIX_FWD_STDDIAG_CHANNEL; } /* setup the byte object */ p = PMIX_NEW(prte_iof_deliver_t); PMIX_XFER_PROCID(&p->source, &proct->name); p->bo.bytes = (char*)malloc(numbytes); memcpy(p->bo.bytes, data, numbytes); p->bo.size = numbytes; prc = PMIx_server_IOF_deliver(&p->source, pchan, &p->bo, NULL, 0, lkcbfunc, (void*)p); if (PMIX_SUCCESS != prc) { PMIX_ERROR_LOG(prc); PMIX_RELEASE(p); } /* prep the buffer */ PMIX_DATA_BUFFER_CREATE(buf); /* pack the stream first - we do this so that flow control messages can * consist solely of the tag */ rc = PMIx_Data_pack(NULL, buf, &rev->tag, 1, PMIX_UINT16); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } /* pack name of process that gave us this data */ rc = PMIx_Data_pack(NULL, buf, &proct->name, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } /* pack the #bytes we read */ rc = PMIx_Data_pack(NULL, buf, &numbytes, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } /* pack the data - only pack the #bytes we read! */ rc = PMIx_Data_pack(NULL, buf, data, numbytes, PMIX_BYTE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto CLEAN_RETURN; } /* start non-blocking RML call to forward received data */ PMIX_OUTPUT_VERBOSE((1, prte_iof_base_framework.framework_output, "%s iof:prted:read handler sending %d bytes to HNP", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), numbytes)); PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, buf, PRTE_RML_TAG_IOF_HNP); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); } /* re-add the event */ PRTE_IOF_READ_ACTIVATE(rev); return; CLEAN_RETURN: /* must be an error, or zero bytes were read indicating that the * proc terminated this IOF channel - either way, release the * corresponding event. This deletes the read event and closes * the file descriptor */ if (rev->tag & PRTE_IOF_STDOUT) { if (NULL != proct->revstdout) { PMIX_RELEASE(proct->revstdout); } } else if (rev->tag & PRTE_IOF_STDERR) { if (NULL != proct->revstderr) { PMIX_RELEASE(proct->revstderr); } } /* check to see if they are all done */ if (NULL == proct->revstdout && NULL == proct->revstderr) { /* this proc's iof is complete */ PRTE_ACTIVATE_PROC_STATE(&proct->name, PRTE_PROC_STATE_IOF_COMPLETE); } if (NULL != buf) { PMIX_DATA_BUFFER_RELEASE(buf); } return; } prrte-4.1.1~rc2/src/mca/iof/prted/Makefile.am0000664000175000017500000000347115203542132021122 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_iof_prted_DSO component_noinst = component_install = prte_mca_iof_prted.la else component_noinst = libprtemca_iof_prted.la component_install = endif prted_SOURCES = \ iof_prted.c \ iof_prted.h \ iof_prted_component.c \ iof_prted_read.c \ iof_prted_receive.c mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_iof_prted_la_SOURCES = $(prted_SOURCES) prte_mca_iof_prted_la_LDFLAGS = -module -avoid-version prte_mca_iof_prted_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_iof_prted_la_SOURCES = $(prted_SOURCES) libprtemca_iof_prted_la_LIBADD = libprtemca_iof_prted_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/errmgr/0000775000175000017500000000000015203542132016464 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/errmgr/dvm/0000775000175000017500000000000015203542132017252 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/errmgr/dvm/owner.txt0000664000175000017500000000023715203542132021147 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: active prrte-4.1.1~rc2/src/mca/errmgr/dvm/errmgr_dvm.h0000664000175000017500000000177115203542132021575 0ustar alastairalastair/* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef prte_errmgr_dvm_EXPORT_H #define prte_errmgr_dvm_EXPORT_H #include "prte_config.h" #include "src/mca/errmgr/errmgr.h" BEGIN_C_DECLS /* * Local Component structures */ PRTE_MODULE_EXPORT extern prte_errmgr_base_component_t prte_mca_errmgr_dvm_component; PRTE_EXPORT extern prte_errmgr_base_module_t prte_errmgr_dvm_module; END_C_DECLS #endif /* prte_errmgr_dvm_EXPORT_H */ prrte-4.1.1~rc2/src/mca/errmgr/dvm/errmgr_dvm.c0000664000175000017500000007205515203542132021573 0ustar alastairalastair/* * Copyright (c) 2009-2011 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2010-2017 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2011 Oracle and/or all its affiliates. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #ifdef HAVE_SYS_WAIT_H # include #endif #include "src/pmix/pmix-internal.h" #include #include #include "src/pmix/pmix-internal.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/mca/ess/ess.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/base/base.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/mca/state/base/base.h" #include "src/threads/pmix_threads.h" #include "src/util/error_strings.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_quit.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/mca/errmgr/errmgr.h" #include "errmgr_dvm.h" static int init(void); static int finalize(void); /****************** * dvm module ******************/ prte_errmgr_base_module_t prte_errmgr_dvm_module = { .init = init, .finalize = finalize, .logfn = prte_errmgr_base_log }; /* * Local functions */ static void job_errors(int fd, short args, void *cbdata); static void proc_errors(int fd, short args, void *cbdata); static void check_send_notification(prte_job_t *jdata, prte_proc_t *proc, pmix_status_t event); static int init(void) { /* setup state machine to trap job errors */ prte_state.add_job_state(PRTE_JOB_STATE_ERROR, job_errors); /* set the lost connection state to run at MSG priority so * we can process any last messages from the proc */ prte_state.add_proc_state(PRTE_PROC_STATE_COMM_FAILED, proc_errors); prte_state.add_proc_state(PRTE_PROC_STATE_ERROR, proc_errors); return PRTE_SUCCESS; } static int finalize(void) { return PRTE_SUCCESS; } static void _terminate_job(pmix_nspace_t jobid) { pmix_pointer_array_t procs; prte_proc_t pobj; PMIX_CONSTRUCT(&procs, pmix_pointer_array_t); pmix_pointer_array_init(&procs, 1, 1, 1); PMIX_CONSTRUCT(&pobj, prte_proc_t); PMIX_LOAD_PROCID(&pobj.name, jobid, PMIX_RANK_WILDCARD); pmix_pointer_array_add(&procs, &pobj); prte_plm.terminate_procs(&procs); PMIX_DESTRUCT(&procs); PMIX_DESTRUCT(&pobj); } static void job_errors(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_job_state_t jobstate; int32_t rc; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* * if prte is trying to shutdown, just let it */ if (prte_finalizing) { return; } /* if the jdata is NULL, then it is referencing the daemon job */ if (NULL == caddy->jdata) { caddy->jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); PMIX_RETAIN(caddy->jdata); } /* update the state */ jdata = caddy->jdata; jobstate = caddy->job_state; jdata->state = jobstate; pmix_output_verbose(1, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: job %s reported state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), prte_job_state_to_str(jobstate)); if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { if (PRTE_JOB_STATE_FAILED_TO_START == jdata->state || PRTE_JOB_STATE_NEVER_LAUNCHED == jdata->state || PRTE_JOB_STATE_FAILED_TO_LAUNCH == jdata->state || PRTE_JOB_STATE_CANNOT_LAUNCH == jdata->state) { prte_routing_is_enabled = false; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); PMIX_RELEASE(caddy); return; } /* if the daemon job aborted and we haven't heard from everyone yet, * then this could well have been caused by a daemon not finding * a way back to us. In this case, output a message indicating a daemon * died without reporting. Otherwise, say nothing as we * likely already output an error message */ if (PRTE_JOB_STATE_ABORTED == jobstate && jdata->num_procs != jdata->num_reported) { prte_routing_is_enabled = false; pmix_show_help("help-errmgr-base.txt", "failed-daemon", true); } /* there really isn't much else we can do since the problem * is in the DVM itself, so best just to terminate */ jdata->num_terminated = jdata->num_procs; /* activate the terminated state so we can exit */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); PMIX_RELEASE(caddy); return; } /* all other cases involve jobs submitted to the DVM - therefore, * we only inform the submitter of the problem, but do NOT terminate * the DVM itself */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm sending notification of job %s failure to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), PRTE_NAME_PRINT(&jdata->originator))); /* all jobs were spawned by a requestor, so ensure that requestor * has been notified that the spawn completed - otherwise, a quick-failing * job might not generate a spawn response */ rc = prte_pmix_convert_job_state_to_error(jobstate); rc = prte_plm_base_spawn_response(rc, jdata); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* ensure we terminate any processes left running in the DVM */ _terminate_job(jdata->nspace); /* if the job never launched, then we need to let the * state machine know this job failed - it has no * other means of being alerted since no proc states * will be triggered */ if (PRTE_JOB_STATE_FAILED_TO_START == jdata->state || PRTE_JOB_STATE_NEVER_LAUNCHED == jdata->state || PRTE_JOB_STATE_FAILED_TO_LAUNCH == jdata->state || PRTE_JOB_STATE_ALLOC_FAILED == jdata->state || PRTE_JOB_STATE_MAP_FAILED == jdata->state || PRTE_JOB_STATE_CANNOT_LAUNCH == jdata->state) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); } /* cleanup */ PMIX_RELEASE(caddy); } static void proc_errors(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_proc_t *pptr, *proct; pmix_proc_t *proc = &caddy->name; prte_proc_state_t state = caddy->proc_state; int i; int32_t i32, *i32ptr; bool flag; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); pmix_output_verbose(1, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: for proc %s state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)); /* get the job object */ if (prte_finalizing || NULL == (jdata = prte_get_job_data_object(proc->nspace))) { /* could be a race condition */ PMIX_RELEASE(caddy); return; } pptr = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank); if (NULL == pptr) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); goto cleanup; } if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { /* NOTE: this is a daemon process that had the error */ /* we MUST handle a communication failure with special care to * avoid normal termination issues */ if (PRTE_PROC_STATE_COMM_FAILED == state || PRTE_PROC_STATE_HEARTBEAT_FAILED == state || PRTE_PROC_STATE_UNABLE_TO_SEND_MSG == state || PRTE_PROC_STATE_FAILED_TO_START == state) { /* if this is my own connection, ignore it */ if (PRTE_PROC_MY_NAME->rank == proc->rank) { PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s Comm failure on my own connection - ignoring it", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } /* mark the daemon as gone */ PRTE_FLAG_UNSET(pptr, PRTE_PROC_FLAG_ALIVE); /* update the state */ pptr->state = state; /* adjust our num_procs */ --prte_process_info.num_daemons; /* if we have ordered prteds to terminate or abort * is in progress, record it */ if (prte_prteds_term_ordered || prte_abnormal_term_ordered) { PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s Comm failure: daemons terminating - recording daemon %s as gone", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc))); /* remove from dependent routes, if it is one */ prte_rml_route_lost(proc->rank); /* if all my routes and local children are gone, then terminate ourselves */ if (0 == pmix_list_get_size(&prte_rml_base.children)) { for (i = 0; i < prte_local_children->size; i++) { proct = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL != proct && PRTE_FLAG_TEST(pptr, PRTE_PROC_FLAG_ALIVE) && proct->state < PRTE_PROC_STATE_UNTERMINATED) { /* at least one is still alive */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s Comm failure: at least one proc (%s) still alive", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proct->name))); goto cleanup; } } /* call our appropriate exit procedure */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr_dvm: all routes and children gone - ordering exit", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); } else { PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s Comm failure: %d routes remain alive", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) pmix_list_get_size(&prte_rml_base.children))); } goto cleanup; } PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s Comm failure: daemon %s - aborting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc))); /* record the first one to fail */ if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { if (PRTE_PROC_STATE_FAILED_TO_START != state) { /* output an error message so the user knows what happened */ pmix_show_help("help-errmgr-base.txt", "node-died", true, PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), prte_process_info.nodename, PRTE_NAME_PRINT(proc), pptr->node->name); } /* mark the daemon job as failed */ jdata->state = PRTE_JOB_STATE_COMM_FAILED; /* point to the lowest rank to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); /* update our exit code */ jdata->exit_code = pptr->exit_code; /* just in case the exit code hadn't been set, do it here - this * won't override any reported exit code */ if (0 == jdata->exit_code) { jdata->exit_code = PRTE_ERR_COMM_FAILURE; } } } else { pmix_output(0, "UNSUPPORTED DAEMON ERROR STATE: %s", prte_proc_state_to_str(state)); } /* since communications have failed, we have to simply force ourselves * to terminate as we cannot rely on the routing tree to get messages * out to other daemons */ prte_abnormal_term_ordered = true; PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); goto cleanup; } /* update the proc state - can get multiple reports on a proc * depending on circumstances, so ensure we only do this once */ if (pptr->state < PRTE_PROC_STATE_TERMINATED) { pptr->state = state; } /* if we were ordered to terminate, mark this proc as dead and see if * any of our routes or local children remain alive - if not, then * terminate ourselves. */ if (prte_prteds_term_ordered) { for (i = 0; i < prte_local_children->size; i++) { proct = (prte_proc_t*)pmix_pointer_array_get_item(prte_local_children, i); if (NULL != proct) { if (PRTE_FLAG_TEST(proct, PRTE_PROC_FLAG_ALIVE)) { goto keep_going; } } } /* if all my routes and children are gone, then terminate ourselves nicely (i.e., this is a normal termination) */ if (0 == pmix_list_get_size(&prte_rml_base.children)) { pmix_output_verbose(2, prte_errmgr_base_framework.framework_output, "%s errmgr:default:dvm all routes gone - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); } } keep_going: /* always mark the waitpid as having fired */ PRTE_ACTIVATE_PROC_STATE(&pptr->name, PRTE_PROC_STATE_WAITPID_FIRED); /* if this is a remote proc, we won't hear anything more about it * as the default behavior would be to terminate the job. So be sure to * mark the IOF as having completed too so we correctly mark this proc * as dead and notify everyone as required */ if (!PRTE_FLAG_TEST(pptr, PRTE_PROC_FLAG_LOCAL)) { PRTE_ACTIVATE_PROC_STATE(&pptr->name, PRTE_PROC_STATE_IOF_COMPLETE); } /* simplify later checks */ flag = (prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, NULL, PMIX_BOOL)); /* ensure we record the failed proc properly so we can report * the error once we terminate */ switch (state) { case PRTE_PROC_STATE_KILLED_BY_CMD: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s killed by cmd", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)); /* we ordered this proc to die, so it isn't an abnormal termination * and we don't flag it as such */ if (jdata->num_terminated >= jdata->num_procs) { /* this job has terminated */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); } else if (flag) { /* at least one proc survives - send out a notification if one is requested */ check_send_notification(jdata, pptr, PMIX_ERR_PROC_KILLED_BY_CMD); // recover the resources used by this proc prte_state_base_recover_resources(jdata, pptr); } break; case PRTE_PROC_STATE_ABORTED_BY_SIG: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s aborted by signal", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)); if (flag) { /* send out a notification if one is requested */ check_send_notification(jdata, pptr, PMIX_ERR_PROC_ABORTED_BY_SIG); // recover the resources used by this proc prte_state_base_recover_resources(jdata, pptr); } else { if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { jdata->state = PRTE_JOB_STATE_ABORTED_BY_SIG; /* point to the first rank to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); jdata->exit_code = pptr->exit_code; _terminate_job(jdata->nspace); } } break; case PRTE_PROC_STATE_TERM_WO_SYNC: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s terminated without sync", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc)); if (flag) { /* send out a notification if one is requested */ check_send_notification(jdata, pptr, PMIX_ERR_PROC_TERM_WO_SYNC); // recover the resources used by this proc prte_state_base_recover_resources(jdata, pptr); } else { if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { jdata->state = PRTE_JOB_STATE_ABORTED_WO_SYNC; /* point to the first rank to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); jdata->exit_code = pptr->exit_code; /* send out a notification is one is requested */ check_send_notification(jdata, pptr, PMIX_ERR_PROC_TERM_WO_SYNC); /* now treat a special case - if the proc exit'd without a required * sync, it may have done so with a zero exit code. We want to ensure * that the user realizes there was an error, so in this -one- case, * we overwrite the process' exit code with the default error code */ if (0 == jdata->exit_code) { jdata->exit_code = PRTE_ERROR_DEFAULT_EXIT_CODE; } /* kill the job */ _terminate_job(jdata->nspace); } } break; case PRTE_PROC_STATE_FAILED_TO_START: case PRTE_PROC_STATE_FAILED_TO_LAUNCH: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)); if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { if (PRTE_PROC_STATE_FAILED_TO_START) { jdata->state = PRTE_JOB_STATE_FAILED_TO_START; } else { jdata->state = PRTE_JOB_STATE_FAILED_TO_LAUNCH; } /* point to the first rank to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* update our exit code */ jdata->exit_code = pptr->exit_code; /* just in case the exit code hadn't been set, do it here - this * won't override any reported exit code */ if (0 == jdata->exit_code) { jdata->exit_code = PRTE_ERR_FAILED_TO_START; } /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); /* kill the job */ _terminate_job(jdata->nspace); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FAILED_TO_START); } /* if this was a daemon, report it */ if (PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { /* output a message indicating we failed to launch a daemon */ pmix_show_help("help-errmgr-base.txt", "failed-daemon-launch", true, prte_tool_basename); } break; case PRTE_PROC_STATE_CALLED_ABORT: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s called abort with exit code %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), pptr->exit_code); /* this proc ordered the job to abort */ if (flag) { /* send out a notification if one is requested */ check_send_notification(jdata, pptr, PMIX_ERR_PROC_REQUESTED_ABORT); // recover the resources used by this proc prte_state_base_recover_resources(jdata, pptr); } else { if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { jdata->state = PRTE_JOB_STATE_CALLED_ABORT; /* point to the first proc to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); jdata->exit_code = pptr->exit_code; /* kill the job */ _terminate_job(jdata->nspace); } } break; case PRTE_PROC_STATE_TERM_NON_ZERO: pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s exited with non-zero status %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), pptr->exit_code); jdata->exit_code = pptr->exit_code; PRTE_FLAG_UNSET(pptr, PRTE_PROC_FLAG_ALIVE); /* track the number of non-zero exits */ i32 = 0; i32ptr = &i32; prte_get_attribute(&jdata->attributes, PRTE_JOB_NUM_NONZERO_EXIT, (void **) &i32ptr, PMIX_INT32); ++i32; prte_set_attribute(&jdata->attributes, PRTE_JOB_NUM_NONZERO_EXIT, PRTE_ATTR_LOCAL, i32ptr, PMIX_INT32); if (flag && prte_get_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, NULL, PMIX_BOOL)) { check_send_notification(jdata, pptr, PMIX_ERR_EXIT_NONZERO_TERM); // recover the resources used by this proc prte_state_base_recover_resources(jdata, pptr); } else { if (!PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { jdata->state = PRTE_JOB_STATE_NON_ZERO_TERM; /* point to the first rank to cause the problem */ prte_set_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, PRTE_ATTR_LOCAL, pptr, PMIX_POINTER); /* retain the object so it doesn't get free'd */ PMIX_RETAIN(pptr); PRTE_FLAG_SET(jdata, PRTE_JOB_FLAG_ABORTED); /* kill the job */ _terminate_job(jdata->nspace); } } break; default: /* shouldn't get this, but terminate job if required */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr:dvm: proc %s default error %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state))); if (jdata->num_terminated == jdata->num_procs) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); } break; } cleanup: PMIX_RELEASE(caddy); } static void check_send_notification(prte_job_t *jdata, prte_proc_t *proc, pmix_status_t event) { int rc; pmix_info_t *info; size_t ninfo; pmix_proc_t target; pmix_data_buffer_t pbkt; pmix_data_range_t range = PMIX_RANGE_CUSTOM; pmix_output_verbose(5, prte_state_base_framework.framework_output, "%s errmgr:dvm:sending notification %s affected proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIx_Error_string(event), PRTE_NAME_PRINT(&proc->name)); if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, NULL, PMIX_BOOL) || prte_dvm_abort_ordered) { return; } /* we checked for termination due to the specific error we encountered, but * it is possible that we received another type of error that wouldn't have * resulted in terminating the job. Thus, we still check to see if the job * is being aborted before sending anything out */ if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) { /* this job has already been aborted, so we don't need to notify * about the fate of any proc within it */ return; } /* notify the other procs of the termination */ PMIX_LOAD_PROCID(&target, jdata->nspace, PMIX_RANK_WILDCARD); /* pack the info for sending */ PMIX_DATA_BUFFER_CONSTRUCT(&pbkt); /* we need to add a flag indicating this came from an invalid proc so that we will * inject it into our own PMIx server library */ rc = PMIx_Data_pack(NULL, &pbkt, &PRTE_NAME_INVALID->rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } /* pack the status code */ rc = PMIx_Data_pack(NULL, &pbkt, &event, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } /* pack the source - it cannot be me as that will cause * the pmix server to upcall the event back to me */ rc = PMIx_Data_pack(NULL, &pbkt, &proc->name, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } /* pack the range */ rc = PMIx_Data_pack(NULL, &pbkt, &range, 1, PMIX_DATA_RANGE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } /* setup the info */ if (-1 != proc->exit_code) { ninfo = 3; } else { ninfo = 2; } PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_EVENT_AFFECTED_PROC, &proc->name, PMIX_PROC); PMIX_INFO_LOAD(&info[1], PMIX_EVENT_CUSTOM_RANGE, &target, PMIX_PROC); if (-1 != proc->exit_code) { PMIX_INFO_LOAD(&info[2], PMIX_EXIT_CODE, &proc->exit_code, PMIX_INT); } /* pack the number of infos */ rc = PMIx_Data_pack(NULL, &pbkt, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } /* pack the infos themselves */ rc = PMIx_Data_pack(NULL, &pbkt, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); return; } PMIX_INFO_FREE(info, ninfo); /* xcast it to everyone */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_NOTIFICATION, &pbkt))) { PRTE_ERROR_LOG(rc); } PMIX_DATA_BUFFER_DESTRUCT(&pbkt); } prrte-4.1.1~rc2/src/mca/errmgr/dvm/errmgr_dvm_component.c0000664000175000017500000000600315203542132023643 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/util/pmix_output.h" #include "errmgr_dvm.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/mca/errmgr/errmgr.h" /* * Public string for version number */ const char *prte_mca_errmgr_dvm_component_version_string = "PRTE ERRMGR dvm MCA component version " PRTE_VERSION; /* * Local functionality */ static int dvm_register(void); static int dvm_open(void); static int dvm_close(void); static int dvm_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointer to our public functions in it */ prte_errmgr_base_component_t prte_mca_errmgr_dvm_component = { /* Handle the general mca_component_t struct containing * meta information about the component dvm */ .base_version = { PRTE_ERRMGR_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "dvm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = dvm_open, .pmix_mca_close_component = dvm_close, .pmix_mca_query_component = dvm_component_query, .pmix_mca_register_component_params = dvm_register, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, errmgr, dvm) static int my_priority; static int dvm_register(void) { pmix_mca_base_component_t *c = &prte_mca_errmgr_dvm_component.base_version; my_priority = 1000; (void) pmix_mca_base_component_var_register(c, "priority", "Priority of the dvm errmgr component", PMIX_MCA_BASE_VAR_TYPE_INT, &my_priority); return PRTE_SUCCESS; } static int dvm_open(void) { return PRTE_SUCCESS; } static int dvm_close(void) { return PRTE_SUCCESS; } static int dvm_component_query(pmix_mca_base_module_t **module, int *priority) { /* used by DVM masters */ if (PRTE_PROC_IS_MASTER) { *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_errmgr_dvm_module; return PRTE_SUCCESS; } *module = NULL; *priority = -1; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/errmgr/dvm/Makefile.am0000664000175000017500000000215515203542132021311 0ustar alastairalastair# # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2016-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ errmgr_dvm.h \ errmgr_dvm_component.c \ errmgr_dvm.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_errmgr_dvm_DSO component_noinst = component_install = prte_mca_errmgr_dvm.la else component_noinst = libprtemca_errmgr_dvm.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_errmgr_dvm_la_SOURCES = $(sources) prte_mca_errmgr_dvm_la_LDFLAGS = -module -avoid-version prte_mca_errmgr_dvm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_errmgr_dvm_la_SOURCES =$(sources) libprtemca_errmgr_dvm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/errmgr/base/0000775000175000017500000000000015203542132017376 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/errmgr/base/errmgr_base_frame.c0000664000175000017500000000631315203542132023207 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #include "src/class/pmix_list.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/base/static-components.h" /* Public module provides a wrapper around previous functions */ prte_errmgr_base_module_t prte_errmgr_default_fns = { .init = NULL, /* init */ .finalize = NULL, /* finalize */ .logfn = prte_errmgr_base_log }; /* NOTE: ABSOLUTELY MUST initialize this * struct to include the log function as it * gets called even if the errmgr hasn't been * opened yet due to error */ prte_errmgr_base_module_t prte_errmgr = { .logfn = prte_errmgr_base_log }; static int prte_errmgr_base_close(void) { /* Close selected component */ if (NULL != prte_errmgr.finalize) { prte_errmgr.finalize(); } /* always leave a default set of fn pointers */ prte_errmgr = prte_errmgr_default_fns; return pmix_mca_base_framework_components_close(&prte_errmgr_base_framework, NULL); } /** * * Function for finding and opening either all MCA components, or the one * * that was specifically requested via a MCA parameter. * */ static int prte_errmgr_base_open(pmix_mca_base_open_flag_t flags) { /* load the default fns */ prte_errmgr = prte_errmgr_default_fns; /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_errmgr_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, errmgr, "PRTE Error Manager", NULL, prte_errmgr_base_open, prte_errmgr_base_close, prte_errmgr_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); prrte-4.1.1~rc2/src/mca/errmgr/base/owner.txt0000664000175000017500000000024415203542132021271 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/errmgr/base/help-errmgr-base.txt0000664000175000017500000000602715203542132023300 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # # [failed-daemon-launch] %s was unable to reliably start one or more daemons. This usually is caused by: * the inability to resolve specified hostnames - please check above for any output regarding this possibility * not finding the required libraries and/or binaries on one or more nodes. Please check your PATH and LD_LIBRARY_PATH settings, or configure OMPI with --enable-prun-prefix-by-default * lack of authority to execute on one or more specified nodes. Please verify your allocation and authorities. * the inability to write startup files into /tmp (--tmpdir/prte_tmpdir_base). Please check with your sys admin to determine the correct location to use. * compilation of the orted with dynamic libraries when static are required (e.g., on Cray). Please check your configure cmd line and consider using one of the contrib/platform definitions for your system type. * an inability to create a connection back to mpirun due to a lack of common network interfaces and/or no route found between them. Please check network connectivity (including firewalls and network routing requirements). # [failed-daemon] An PRTE daemon has unexpectedly failed after launch and before communicating back to mpirun. This could be caused by a number of factors, including an inability to create a connection back to mpirun due to a lack of common network interfaces and/or no route found between them. Please check network connectivity (including firewalls and network routing requirements). # [node-died] PRTE has lost communication with a remote daemon. HNP daemon : %s on node %s Remote daemon: %s on node %s This is usually due to either a failure of the TCP network connection to the node, or possibly an internal failure of the daemon itself. We cannot recover from this failure, and therefore will terminate the job. # [simple-message] An internal error has occurred in PRTE: %s This is something that should be reported to the developers. prrte-4.1.1~rc2/src/mca/errmgr/base/errmgr_base_fns.c0000664000175000017500000000563615203542132022712 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #if HAVE_SYS_TYPES_H # include #endif /* HAVE_SYS_TYPES_H */ #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #if HAVE_SYS_TYPES_H # include #endif /* HAVE_SYS_TYPES_H */ #if HAVE_SYS_STAT_H # include #endif /* HAVE_SYS_STAT_H */ #ifdef HAVE_DIRENT_H # include #endif /* HAVE_DIRENT_H */ #include #include #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_wait.h" #include "src/runtime/runtime.h" #include "src/mca/ess/ess.h" #include "src/mca/odls/odls.h" #include "src/mca/plm/plm.h" #include "src/mca/state/state.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/mca/errmgr/errmgr.h" /* * Public interfaces */ void prte_errmgr_base_log(int error_code, char *filename, int line) { char *errstring = NULL; errstring = (char *) PRTE_ERROR_NAME(error_code); if (NULL == errstring) { /* if the error is silent, say nothing */ return; } pmix_output(0, "%s PRTE_ERROR_LOG: %s in file %s at line %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), errstring, filename, line); } prrte-4.1.1~rc2/src/mca/errmgr/base/errmgr_base_select.c0000664000175000017500000000446015203542132023375 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2012-2015 Los Alamos National Security, Inc. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_output.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" int prte_errmgr_base_select(void) { int exit_status = PRTE_SUCCESS; prte_errmgr_base_component_t *best_component = NULL; prte_errmgr_base_module_t *best_module = NULL; /* * Select the best component */ if (PRTE_SUCCESS != pmix_mca_base_select("errmgr", prte_errmgr_base_framework.framework_output, &prte_errmgr_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL)) { /* This will only happen if no component was selected */ exit_status = PRTE_ERROR; goto cleanup; } /* Save the winner */ prte_errmgr = *best_module; /* Initialize the winner */ if (NULL != prte_errmgr.init) { if (PRTE_SUCCESS != prte_errmgr.init()) { exit_status = PRTE_ERROR; goto cleanup; } } cleanup: return exit_status; } prrte-4.1.1~rc2/src/mca/errmgr/base/errmgr_private.h0000664000175000017500000000335515203542132022605 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2010 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_ERRMGR_PRIVATE_H #define PRTE_MCA_ERRMGR_PRIVATE_H /* * includes */ #include "prte_config.h" #include "constants.h" #include "types.h" #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" #include "src/mca/errmgr/errmgr.h" /* * Functions for use solely within the ERRMGR framework */ BEGIN_C_DECLS /* declare the base default module */ PRTE_EXPORT extern prte_errmgr_base_module_t prte_errmgr_default_fns; /* * Base functions */ PRTE_EXPORT void prte_errmgr_base_log(int error_code, char *filename, int line); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/errmgr/base/base.h0000664000175000017500000000265415203542132020470 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_ERRMGR_BASE_H #define PRTE_MCA_ERRMGR_BASE_H /* * includes */ #include "prte_config.h" #include "constants.h" #include "src/class/pmix_list.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/mca.h" BEGIN_C_DECLS /* * MCA Framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_errmgr_base_framework; /* select a component */ PRTE_EXPORT int prte_errmgr_base_select(void); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/errmgr/base/Makefile.am0000664000175000017500000000232415203542132021433 0ustar alastairalastair# # Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. # Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ base/help-errmgr-base.txt headers += \ base/errmgr_private.h \ base/base.h libprtemca_errmgr_la_SOURCES += \ base/errmgr_base_select.c \ base/errmgr_base_frame.c \ base/errmgr_base_fns.c prrte-4.1.1~rc2/src/mca/errmgr/errmgr.h0000664000175000017500000001042715203542132020137 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014 NVIDIA Corporation. All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The PRTE Error and Recovery Manager (ErrMgr) * * This framework is the logically central clearing house for process/daemon * state updates. In particular when a process fails and another process detects * it, then that information is reported through this framework. This framework * then (depending on the active component) decides how to handle the failure. * * For example, if a process fails this may activate an automatic recovery * of the process from a previous checkpoint, or initial state. Conversely, * the active component could decide not to continue the job, and request that * it be terminated. The error and recovery policy is determined by individual * components within this framework. * */ #ifndef PRTE_MCA_ERRMGR_H #define PRTE_MCA_ERRMGR_H /* * includes */ #include "prte_config.h" #include "constants.h" #include "types.h" #include "src/pmix/pmix-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/class/pmix_object.h" #include "src/class/pmix_pointer_array.h" #include "src/util/error.h" #include "src/util/pmix_output.h" #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS /* * Macro definitions */ /* * Thess macros and associated error name array are used to output intelligible error * messages. */ #define PRTE_ERROR_NAME(n) prte_strerror(n) /* * Framework Interfaces */ /** * Module initialization function. * * @retval PRTE_SUCCESS The operation completed successfully * @retval PRTE_ERROR An unspecifed error occurred */ typedef int (*prte_errmgr_base_module_init_fn_t)(void); /** * Module finalization function. * * @retval PRTE_SUCCESS The operation completed successfully * @retval PRTE_ERROR An unspecifed error occurred */ typedef int (*prte_errmgr_base_module_finalize_fn_t)(void); /** * This is not part of any module so it can be used at any time! */ typedef void (*prte_errmgr_base_module_log_fn_t)(int error_code, char *filename, int line); /* * Module Structure */ struct prte_errmgr_base_module_2_3_0_t { /** Initialization Function */ prte_errmgr_base_module_init_fn_t init; /** Finalization Function */ prte_errmgr_base_module_finalize_fn_t finalize; prte_errmgr_base_module_log_fn_t logfn; }; typedef struct prte_errmgr_base_module_2_3_0_t prte_errmgr_base_module_2_3_0_t; typedef prte_errmgr_base_module_2_3_0_t prte_errmgr_base_module_t; PRTE_EXPORT extern prte_errmgr_base_module_t prte_errmgr; /* * ErrMgr Component */ struct prte_errmgr_base_component_3_0_0_t { /** MCA base component */ pmix_mca_base_component_t base_version; /** Verbosity Level */ int verbose; /** Output Handle for pmix_output */ int output_handle; /** Default Priority */ int priority; }; typedef struct prte_errmgr_base_component_3_0_0_t prte_errmgr_base_component_3_0_0_t; typedef prte_errmgr_base_component_3_0_0_t prte_errmgr_base_component_t; /* * Macro for use in components that are of type errmgr */ #define PRTE_ERRMGR_BASE_VERSION_3_0_0 PRTE_MCA_BASE_VERSION_3_0_0("errmgr", 3, 0, 0) END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/errmgr/Makefile.am0000664000175000017500000000241215203542132020517 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_errmgr.la libprtemca_errmgr_la_SOURCES = # local files headers = errmgr.h libprtemca_errmgr_la_SOURCES += $(headers) # pkgdata setup EXTRA_DIST = # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/errmgr/prted/0000775000175000017500000000000015203542132017602 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/errmgr/prted/owner.txt0000664000175000017500000000024415203542132021475 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/errmgr/prted/errmgr_prted_component.c0000664000175000017500000000615315203542132024531 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/util/pmix_output.h" #include "errmgr_prted.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/errmgr.h" /* * Public string for version number */ const char *prte_mca_errmgr_prted_component_version_string = "PRTE ERRMGR prted MCA component version " PRTE_VERSION; /* * Local functionality */ static int errmgr_prted_register(void); static int errmgr_prted_open(void); static int errmgr_prted_close(void); static int errmgr_prted_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointer to our public functions in it */ prte_errmgr_base_component_t prte_mca_errmgr_prted_component = { /* Handle the general mca_component_t struct containing * meta information about the component itprted */ .base_version = { PRTE_ERRMGR_BASE_VERSION_3_0_0, /* Component name and version */ .pmix_mca_component_name = "prted", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = errmgr_prted_open, .pmix_mca_close_component = errmgr_prted_close, .pmix_mca_query_component = errmgr_prted_component_query, .pmix_mca_register_component_params = errmgr_prted_register, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, errmgr, prted) static int my_priority; static int errmgr_prted_register(void) { pmix_mca_base_component_t *c = &prte_mca_errmgr_prted_component.base_version; my_priority = 1000; (void) pmix_mca_base_component_var_register(c, "priority", "Priority of the prted errmgr component", PMIX_MCA_BASE_VAR_TYPE_INT, &my_priority); return PRTE_SUCCESS; } static int errmgr_prted_open(void) { return PRTE_SUCCESS; } static int errmgr_prted_close(void) { return PRTE_SUCCESS; } static int errmgr_prted_component_query(pmix_mca_base_module_t **module, int *priority) { if (PRTE_PROC_IS_DAEMON) { /* we are the default component for daemons */ *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_errmgr_prted_module; return PRTE_SUCCESS; } *priority = -1; *module = NULL; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/errmgr/prted/errmgr_prted.h0000664000175000017500000000151015203542132022444 0ustar alastairalastair/* * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef MCA_ERRMGR_prted_EXPORT_H #define MCA_ERRMGR_prted_EXPORT_H #include "prte_config.h" #include "src/mca/errmgr/errmgr.h" BEGIN_C_DECLS /* * Local Component structures */ PRTE_MODULE_EXPORT extern prte_errmgr_base_component_t prte_mca_errmgr_prted_component; PRTE_EXPORT extern prte_errmgr_base_module_t prte_errmgr_prted_module; END_C_DECLS #endif /* MCA_ERRMGR_prted_EXPORT_H */ prrte-4.1.1~rc2/src/mca/errmgr/prted/errmgr_prted.c0000664000175000017500000010316615203542132022451 0ustar alastairalastair/* * Copyright (c) 2009-2010 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2004-2023 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/util/pmix_output.h" #include "src/util/pmix_printf.h" #include "src/threads/pmix_threads.h" #include "src/util/error_strings.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/ess/ess.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/plm_types.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/mca/errmgr/base/base.h" #include "src/mca/errmgr/base/errmgr_private.h" #include "src/mca/errmgr/errmgr.h" #include "errmgr_prted.h" /* * Module functions: Global */ static int init(void); static int finalize(void); static void prted_abort(int error_code, char *fmt, ...); /****************** * prted module ******************/ prte_errmgr_base_module_t prte_errmgr_prted_module = { .init = init, .finalize = finalize, .logfn = prte_errmgr_base_log }; /* Local functions */ static bool any_live_children(pmix_nspace_t job); static int pack_state_update(pmix_data_buffer_t *alert, prte_job_t *jobdat); static int pack_state_for_proc(pmix_data_buffer_t *alert, prte_proc_t *child); static void failed_start(prte_job_t *jobdat); static void killprocs(pmix_nspace_t job, pmix_rank_t vpid); static void job_errors(int fd, short args, void *cbdata); static void proc_errors(int fd, short args, void *cbdata); /************************ * API Definitions ************************/ static int init(void) { /* setup state machine to trap job errors */ prte_state.add_job_state(PRTE_JOB_STATE_ERROR, job_errors); /* set the lost connection state to run at MSG priority so * we can process any last messages from the proc */ prte_state.add_proc_state(PRTE_PROC_STATE_COMM_FAILED, proc_errors); /* setup state machine to trap proc errors */ prte_state.add_proc_state(PRTE_PROC_STATE_ERROR, proc_errors); return PRTE_SUCCESS; } static int finalize(void) { return PRTE_SUCCESS; } static void wakeup(int sd, short args, void *cbdata) { PRTE_HIDE_UNUSED_PARAMS(sd, args, cbdata); /* nothing more we can do */ PMIX_ACQUIRE_OBJECT(cbdata); prte_quit(0, 0, NULL); } /* this function only gets called when FORCED_TERMINATE * has been invoked, which means that there is some * internal failure (e.g., to pack/unpack a correct value). * We could just exit, but that doesn't result in any * meaningful error message to the user. Likewise, just * printing something to stdout/stderr won't necessarily * get back to the user. Instead, we will send an error * report to mpirun and give it a chance to order our * termination. In order to ensure we _do_ terminate, * we set a timer - if it fires before we receive the * termination command, then we will exit on our own. This * protects us in the case that the failure is in the * messaging system itself */ static void prted_abort(int error_code, char *fmt, ...) { va_list arglist; char *outmsg = NULL; prte_plm_cmd_flag_t cmd; pmix_data_buffer_t *alert; pmix_rank_t null = PMIX_RANK_INVALID; prte_proc_state_t state = PRTE_PROC_STATE_CALLED_ABORT; prte_timer_t *timer; int rc; /* only do this once */ if (prte_abnormal_term_ordered) { return; } /* set the aborting flag */ prte_abnormal_term_ordered = true; /* If there was a message, construct it */ va_start(arglist, fmt); if (NULL != fmt) { pmix_vasprintf(&outmsg, fmt, arglist); } va_end(arglist); /* use the show-help system to get the message out */ pmix_show_help("help-errmgr-base.txt", "simple-message", true, outmsg); /* tell the HNP we are in distress */ PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &PRTE_PROC_MY_NAME->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack our vpid */ rc = PMIx_Data_pack(NULL, alert, &PRTE_PROC_MY_NAME->rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack our pid */ rc = PMIx_Data_pack(NULL, alert, &prte_process_info.pid, 1, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack our state */ rc = PMIx_Data_pack(NULL, alert, &state, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack our exit code */ rc = PMIx_Data_pack(NULL, alert, &error_code, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* flag that this job is complete so the receiver can know */ rc = PMIx_Data_pack(NULL, alert, &null, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(alert); /* we can't communicate, so give up */ prte_quit(0, 0, NULL); return; } cleanup: /* set a timer for exiting - this also gives the message a chance * to get out! */ if (NULL == (timer = PMIX_NEW(prte_timer_t))) { PRTE_ERROR_LOG(PRTE_ERR_OUT_OF_RESOURCE); return; } timer->tv.tv_sec = 5; timer->tv.tv_usec = 0; prte_event_evtimer_set(prte_event_base, timer->ev, wakeup, NULL); PMIX_POST_OBJECT(timer); prte_event_evtimer_add(timer->ev, &timer->tv); } static void job_errors(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_job_state_t jobstate; int rc; prte_plm_cmd_flag_t cmd; pmix_data_buffer_t *alert; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* * if prte is trying to shutdown, just let it */ if (prte_finalizing) { return; } /* if the jdata is NULL, then it is referencing the daemon job */ if (NULL == caddy->jdata) { caddy->jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); PMIX_RETAIN(caddy->jdata); } /* update the state */ jdata = caddy->jdata; jobstate = caddy->job_state; jdata->state = jobstate; PMIX_OUTPUT_VERBOSE((1, prte_errmgr_base_framework.framework_output, "%s errmgr:prted: job %s repprted error state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), prte_job_state_to_str(jobstate))); switch (jobstate) { case PRTE_JOB_STATE_FAILED_TO_START: failed_start(jdata); break; case PRTE_JOB_STATE_COMM_FAILED: /* kill all local procs */ killprocs(NULL, PMIX_RANK_WILDCARD); /* order termination */ prted_abort(PRTE_ERROR_DEFAULT_EXIT_CODE, "Daemon %s: comm failure", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); goto cleanup; case PRTE_JOB_STATE_HEARTBEAT_FAILED: /* let the HNP handle this */ goto cleanup; default: break; } PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the job info */ if (PMIX_SUCCESS != (rc = pack_state_update(alert, jdata))) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(alert); } cleanup: PMIX_RELEASE(caddy); } static void proc_errors(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; pmix_proc_t *proc = &caddy->name; prte_proc_state_t state = caddy->proc_state; prte_proc_t *child, *ptr; pmix_data_buffer_t *alert; prte_plm_cmd_flag_t cmd; int rc = PRTE_SUCCESS; int i; prte_wait_tracker_t *t2; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors process %s error state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state))); /* * if prte is trying to shutdown, just let it */ if (prte_finalizing) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors finalizing - ignoring error", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } /* if this is a heartbeat failure, let the HNP handle it */ if (PRTE_PROC_STATE_HEARTBEAT_FAILED == state) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors heartbeat failed - ignoring error", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } /* if this was a failed comm, then see if it was to our * lifeline */ if (PRTE_PROC_STATE_LIFELINE_LOST == state || PRTE_PROC_STATE_UNABLE_TO_SEND_MSG == state || PRTE_PROC_STATE_NO_PATH_TO_TARGET == state || PRTE_PROC_STATE_PEER_UNKNOWN == state || PRTE_PROC_STATE_FAILED_TO_CONNECT == state) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted lifeline lost or unable to communicate - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* set our exit status */ PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); /* kill our children */ killprocs(NULL, PMIX_RANK_WILDCARD); /* terminate - our routed children will see * us leave and automatically die */ prte_quit(0, 0, NULL); goto cleanup; } /* get the job object */ if (NULL == (jdata = prte_get_job_data_object(proc->nspace))) { /* must already be complete */ PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors NULL jdata - ignoring error", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } if (PRTE_PROC_STATE_COMM_FAILED == state) { /* if it is our own connection, ignore it */ if (PRTE_EQUAL == prte_util_compare_name_fields(PRTE_NS_CMP_ALL, PRTE_PROC_MY_NAME, proc)) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors comm_failed to self - ignoring error", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } /* was it a daemon? */ if (!PMIX_CHECK_NSPACE(proc->nspace, PRTE_PROC_MY_NAME->nspace)) { /* nope - we can't seem to trust that we will catch the waitpid * in this situation, so push this over to be handled as if * it were a waitpid trigger so we don't create a bunch of * duplicate code */ PMIX_OUTPUT_VERBOSE( (2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors comm_failed to non-daemon - handling as waitpid", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* get the proc_t */ if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); goto cleanup; } /* leave the exit code alone - process this as a waitpid */ t2 = PMIX_NEW(prte_wait_tracker_t); PMIX_RETAIN(child); // protect against race conditions t2->child = child; prte_event_set(prte_event_base, &t2->ev, -1, PRTE_EV_WRITE, prte_odls_base_default_wait_local_proc, t2); prte_event_active(&t2->ev, PRTE_EV_WRITE, 1); goto cleanup; } PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:default:prted daemon %s exited", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc))); if (prte_prteds_term_ordered) { /* are any of my children still alive */ for (i = 0; i < prte_local_children->size; i++) { if (NULL != (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { if (PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_ALIVE)) { PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s errmgr:default:prted[%s(%d)] proc %s is alive", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), __FILE__, __LINE__, PRTE_NAME_PRINT(&child->name))); goto cleanup; } } } /* if all my routes and children are gone, then terminate ourselves nicely (i.e., this is a normal termination) */ if (0 == pmix_list_get_size(&prte_rml_base.children)) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:default:prted all routes gone - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); } else { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:default:prted not exiting, num_routes() == %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) pmix_list_get_size(&prte_rml_base.children))); } } /* if not, then we can continue */ goto cleanup; } if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); goto cleanup; } if (PRTE_PROC_STATE_CALLED_ABORT == state) { /* update the state */ child->state = state; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, NULL, PMIX_BOOL)) { PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* pack only the data for this proc - have to start with the jobid * so the receiver can unpack it correctly */ rc = PMIx_Data_pack(NULL, alert, &proc->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* now pack the child's info */ if (PMIX_SUCCESS != (rc = pack_state_for_proc(alert, child))) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* send it */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr:prted reporting proc %s called abort with " "non-zero status (local procs = %d)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&child->name), jdata->num_local_procs)); PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(alert); } /* mark that we notified the HNP for this job so we don't do it again; * recoverable jobs need to receive every notifications, though. */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } } goto cleanup; } /* if this is not a local proc for this job, we can * ignore this call */ if (!PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_LOCAL)) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted:proc_errors proc is not local - ignoring error", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto cleanup; } PMIX_OUTPUT_VERBOSE( (2, prte_errmgr_base_framework.framework_output, "%s errmgr:prted got state %s for proc %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), prte_proc_state_to_str(state), PRTE_NAME_PRINT(proc))); if (PRTE_PROC_STATE_TERM_NON_ZERO == state) { /* update the state */ child->state = state; /* report this as abnormal termination to the HNP, unless we already have * done so for this job */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, NULL, PMIX_BOOL)) { PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* pack only the data for this proc - have to start with the jobid * so the receiver can unpack it correctly */ rc = PMIx_Data_pack(NULL, alert, &proc->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* now pack the child's info */ if (PMIX_SUCCESS != (rc = pack_state_for_proc(alert, child))) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* send it */ PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr:prted reporting proc %s abnormally terminated with " "non-zero status (local procs = %d)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&child->name), jdata->num_local_procs)); PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_RELEASE(alert); } /* mark that we notified the HNP for this job so we don't do it again; * recoverable jobs need to receive every notifications, though. */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } } /* if the proc has terminated, notify the state machine */ if (PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_IOF_COMPLETE) && PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_WAITPID) && !PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_RECORDED)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } goto cleanup; } if (PRTE_PROC_STATE_FAILED_TO_START == state || PRTE_PROC_STATE_FAILED_TO_LAUNCH == state) { /* update the proc state */ child->state = state; /* count the proc as having "terminated" */ jdata->num_terminated++; /* leave the error report in this case to the * state machine, which will receive notice * when all local procs have attempted to start * so that we send a consolidated error report * back to the HNP */ if (jdata->num_local_procs == jdata->num_terminated) { /* let the state machine know */ if (PRTE_PROC_STATE_FAILED_TO_START == state) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FAILED_TO_START); } else { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FAILED_TO_LAUNCH); } } goto cleanup; } if (PRTE_PROC_STATE_TERMINATED < state) { /* if we were ordered to terminate, see if * any of our routes or local children remain alive - if not, then * terminate ourselves. */ if (prte_prteds_term_ordered) { /* mark the child as no longer alive and update the counters, if necessary. * we have to do this here as we aren't going to send this to the state * machine, and we want to keep the bookkeeping accurate just in case */ if (PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_ALIVE)) { PRTE_FLAG_UNSET(child, PRTE_PROC_FLAG_ALIVE); } if (!PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_RECORDED)) { PRTE_FLAG_SET(child, PRTE_PROC_FLAG_RECORDED); jdata->num_terminated++; } for (i = 0; i < prte_local_children->size; i++) { if (NULL != (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { if (PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_ALIVE)) { goto keep_going; } } } /* if all my routes and children are gone, then terminate ourselves nicely (i.e., this is a normal termination) */ if (0 == pmix_list_get_size(&prte_rml_base.children)) { PMIX_OUTPUT_VERBOSE((2, prte_errmgr_base_framework.framework_output, "%s errmgr:default:prted all routes gone - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); } /* no need to alert the HNP - we are already on our way out */ goto cleanup; } keep_going: /* if the job hasn't completed and the state is abnormally * terminated, then we need to alert the HNP right away - but * only do this once! */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, NULL, PMIX_BOOL)) { PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* pack only the data for this proc - have to start with the jobid * so the receiver can unpack it correctly */ rc = PMIx_Data_pack(NULL, alert, &proc->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } child->state = state; /* now pack the child's info */ if (PMIX_SUCCESS != (rc = pack_state_for_proc(alert, child))) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } pmix_output_verbose(5, prte_errmgr_base_framework.framework_output, "%s errmgr:prted reporting proc %s aborted to HNP (local procs = %d)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&child->name), jdata->num_local_procs); /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); } /* mark that we reported termination of this proc so we * don't do it again */ PRTE_FLAG_SET(child, PRTE_PROC_FLAG_TERM_REPORTED); /* mark that we notified the HNP for this job so we don't do it again; * recoverable jobs need to receive every notifications, though. */ if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_FAIL_NOTIFIED, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); } } /* if the proc has terminated, notify the state machine */ if (PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_IOF_COMPLETE) && PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_WAITPID) && !PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_RECORDED)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } goto cleanup; } /* only other state is terminated - see if anyone is left alive */ if (!any_live_children(proc->nspace)) { PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } /* pack the data for the job */ if (PMIX_SUCCESS != (rc = pack_state_update(alert, jdata))) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); return; } PMIX_OUTPUT_VERBOSE((5, prte_errmgr_base_framework.framework_output, "%s errmgr:prted reporting all procs in %s terminated", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); /* remove all of this job's children from the global list */ for (i = 0; i < prte_local_children->size; i++) { ptr = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL == ptr) { continue; } if (PMIX_CHECK_NSPACE(jdata->nspace, ptr->name.nspace)) { pmix_pointer_array_set_item(prte_local_children, i, NULL); PMIX_RELEASE(ptr); } } /* remove this job from our local job data since it is complete */ PMIX_RELEASE(jdata); /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); } return; } cleanup: PMIX_RELEASE(caddy); } /***************** * Local Functions *****************/ static bool any_live_children(pmix_nspace_t job) { int i; prte_proc_t *child; for (i = 0; i < prte_local_children->size; i++) { if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { continue; } /* is this child part of the specified job? */ if ((PMIX_NSPACE_INVALID(job) || PMIX_CHECK_NSPACE(job, child->name.nspace)) && PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_ALIVE)) { return true; } } /* if we get here, then nobody is left alive from that job */ return false; } static int pack_state_for_proc(pmix_data_buffer_t *alert, prte_proc_t *child) { int rc; /* pack the child's vpid */ rc = PMIx_Data_pack(NULL, alert, &child->name.rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack the pid */ rc = PMIx_Data_pack(NULL, alert, &child->pid, 1, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack its state */ rc = PMIx_Data_pack(NULL, alert, &child->state, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack its exit code */ rc = PMIx_Data_pack(NULL, alert, &child->exit_code, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } return PRTE_SUCCESS; } static int pack_state_update(pmix_data_buffer_t *alert, prte_job_t *jobdat) { int rc, i; prte_proc_t *child; pmix_rank_t null = PMIX_RANK_INVALID; /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &jobdat->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } for (i = 0; i < prte_local_children->size; i++) { if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { continue; } /* if this child is part of the job... */ if (PMIX_CHECK_NSPACE(child->name.nspace, jobdat->nspace)) { if (PMIX_SUCCESS != (rc = pack_state_for_proc(alert, child))) { PMIX_ERROR_LOG(rc); return rc; } } } /* flag that this job is complete so the receiver can know */ rc = PMIx_Data_pack(NULL, alert, &null, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } return PRTE_SUCCESS; } static void failed_start(prte_job_t *jobdat) { int i; prte_proc_t *child; /* set the state */ jobdat->state = PRTE_JOB_STATE_FAILED_TO_START; for (i = 0; i < prte_local_children->size; i++) { if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { continue; } /* is this child part of the specified job? */ if (PMIX_CHECK_NSPACE(child->name.nspace, jobdat->nspace)) { if (PRTE_PROC_STATE_FAILED_TO_START == child->state) { /* this proc never launched - flag that the iof * is complete or else we will hang waiting for * pipes to close that were never opened */ PRTE_FLAG_SET(child, PRTE_PROC_FLAG_IOF_COMPLETE); /* ditto for waitpid */ PRTE_FLAG_SET(child, PRTE_PROC_FLAG_WAITPID); PRTE_ACTIVATE_PROC_STATE(&child->name, PRTE_PROC_STATE_TERMINATED); } } } PMIX_OUTPUT_VERBOSE((1, prte_errmgr_base_framework.framework_output, "%s errmgr:hnp: job %s reported incomplete start", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jobdat->nspace))); return; } static void killprocs(pmix_nspace_t job, pmix_rank_t vpid) { pmix_pointer_array_t cmd; prte_proc_t proc; int rc; if (PMIX_NSPACE_INVALID(job) && PMIX_RANK_WILDCARD == vpid) { if (PRTE_SUCCESS != (rc = prte_odls.kill_local_procs(NULL))) { PRTE_ERROR_LOG(rc); } return; } PMIX_CONSTRUCT(&cmd, pmix_pointer_array_t); PMIX_CONSTRUCT(&proc, prte_proc_t); PMIX_LOAD_PROCID(&proc.name, job, vpid); pmix_pointer_array_add(&cmd, &proc); if (PRTE_SUCCESS != (rc = prte_odls.kill_local_procs(&cmd))) { PRTE_ERROR_LOG(rc); } PMIX_DESTRUCT(&cmd); PMIX_DESTRUCT(&proc); } prrte-4.1.1~rc2/src/mca/errmgr/prted/Makefile.am0000664000175000017500000000220315203542132021633 0ustar alastairalastair# # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ errmgr_prted.h \ errmgr_prted_component.c \ errmgr_prted.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_errmgr_prted_DSO component_noinst = component_install = prte_mca_errmgr_prted.la else component_noinst = libprtemca_errmgr_prted.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_errmgr_prted_la_SOURCES = $(sources) prte_mca_errmgr_prted_la_LDFLAGS = -module -avoid-version prte_mca_errmgr_prted_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_errmgr_prted_la_SOURCES =$(sources) libprtemca_errmgr_prted_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/grpcomm/0000775000175000017500000000000015203542132016632 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/grpcomm/base/0000775000175000017500000000000015203542132017544 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/grpcomm/base/owner.txt0000664000175000017500000000024415203542132021437 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/grpcomm/base/base.h0000664000175000017500000000403115203542132020625 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef MCA_GRPCOMM_BASE_H #define MCA_GRPCOMM_BASE_H /* * includes */ #include "prte_config.h" #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" #include "src/hwloc/hwloc-internal.h" #include "src/mca/base/pmix_mca_base_framework.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/mca.h" #include "src/mca/odls/odls_types.h" #include "src/rml/rml_types.h" #include "src/pmix/pmix-internal.h" /* * Global functions for MCA overall collective open and close */ BEGIN_C_DECLS /* * MCA framework */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_grpcomm_base_framework; /* * Select an available component. */ PRTE_EXPORT int prte_grpcomm_base_select(void); /* * globals that might be needed */ typedef struct { uint32_t context_id; } prte_grpcomm_base_t; PRTE_EXPORT extern prte_grpcomm_base_t prte_grpcomm_base; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/grpcomm/base/grpcomm_base_stubs.c0000664000175000017500000002323115203542132023567 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * */ /* * includes */ #include "prte_config.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/odls/base/base.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "src/threads/pmix_threads.h" #include "src/util/error_strings.h" #include "src/util/name_fns.h" #include "src/util/proc_info.h" static int pack_xcast(prte_grpcomm_signature_t *sig, pmix_data_buffer_t *buffer, pmix_data_buffer_t *message, prte_rml_tag_t tag); static int create_dmns(prte_grpcomm_signature_t *sig, pmix_rank_t **dmns, size_t *ndmns); int prte_grpcomm_API_xcast(prte_grpcomm_signature_t *sig, prte_rml_tag_t tag, pmix_data_buffer_t *msg) { int rc = PRTE_ERROR; pmix_data_buffer_t *buf; prte_grpcomm_base_active_t *active; pmix_rank_t *dmns; size_t ndmns; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base:xcast sending %u bytes to tag %ld", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == msg) ? 0 : (unsigned int) msg->bytes_used, (long) tag)); /* this function does not access any framework-global data, and * so it does not require us to push it into the event library */ /* prep the output buffer */ PMIX_DATA_BUFFER_CREATE(buf); /* create the array of participating daemons */ if (PRTE_SUCCESS != (rc = create_dmns(sig, &dmns, &ndmns))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } /* setup the payload */ if (PRTE_SUCCESS != (rc = pack_xcast(sig, buf, msg, tag))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); if (NULL != dmns) { free(dmns); } return rc; } /* cycle thru the actives and see who can send it */ PMIX_LIST_FOREACH(active, &prte_grpcomm_base.actives, prte_grpcomm_base_active_t) { if (NULL != active->module->xcast) { if (PRTE_SUCCESS == (rc = active->module->xcast(dmns, ndmns, buf))) { break; } } } if (NULL != dmns) { free(dmns); } return rc; } static void grp_construct_stub(int fd, short args, void *cbdata) { prte_pmix_grp_caddy_t *cd = (prte_pmix_grp_caddy_t *) cbdata; int ret = PRTE_SUCCESS; prte_grpcomm_base_active_t *active; prte_grpcomm_coll_t *coll; uint32_t *seq_number; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(cd); PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base:grp_construct stub", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* retrieve an existing tracker, create it if not * already found. The grp_construct module is responsible * for releasing it upon completion of the collective */ ret = pmix_hash_table_get_value_ptr(&prte_grpcomm_base.sig_table, (void *) cd->sig->groupID, strlen(cd->sig->groupID), (void **) &seq_number); if (PMIX_ERR_NOT_FOUND == ret) { seq_number = (uint32_t *) malloc(sizeof(uint32_t)); *seq_number = 0; } else if (PMIX_SUCCESS == ret) { *seq_number = *seq_number + 1; } else { PMIX_OUTPUT((prte_grpcomm_base_framework.framework_output, "%s rpcomm:base:grp_construct cannot get signature from hash table", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_ERROR_LOG(ret); PMIX_RELEASE(cd); return; } ret = pmix_hash_table_set_value_ptr(&prte_grpcomm_base.sig_table, (void *) cd->sig->groupID, strlen(cd->sig->groupID), (void *) seq_number); if (PMIX_SUCCESS != ret) { PMIX_OUTPUT((prte_grpcomm_base_framework.framework_output, "%s rpcomm:base:grp_construct cannot add new signature to hash table", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_ERROR_LOG(ret); PMIX_RELEASE(cd); return; } coll = prte_grpcomm_base_get_tracker(cd->sig, true); if (NULL == coll) { PMIX_RELEASE(cd->sig); PMIX_RELEASE(cd); return; } PMIX_RELEASE(cd->sig); cd->sig = NULL; coll->cbfunc = cd->grpcbfunc; coll->cbdata = cd; /* cycle thru the actives and see who can process it */ PMIX_LIST_FOREACH(active, &prte_grpcomm_base.actives, prte_grpcomm_base_active_t) { if (NULL != active->module->grp_construct) { if (PRTE_SUCCESS == active->module->grp_construct(coll, cd)) { break; } } } } int prte_grpcomm_API_grp_construct(prte_pmix_grp_caddy_t *cd) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base:grp_construct", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* must push this into the event library to ensure we can * access framework-global data safely */ prte_event_set(prte_event_base, &cd->ev, -1, PRTE_EV_WRITE, grp_construct_stub, cd); PMIX_POST_OBJECT(cd); prte_event_active(&cd->ev, PRTE_EV_WRITE, 1); return PRTE_SUCCESS; } static int pack_xcast(prte_grpcomm_signature_t *sig, pmix_data_buffer_t *buffer, pmix_data_buffer_t *message, prte_rml_tag_t tag) { int rc; pmix_data_buffer_t data; bool compressed; pmix_byte_object_t bo; size_t sz; /* setup an intermediate buffer */ PMIX_DATA_BUFFER_CONSTRUCT(&data); /* pass along the signature */ rc = prte_grpcomm_sig_pack(&data, sig); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&data); return rc; } /* pass the final tag */ rc = PMIx_Data_pack(NULL, &data, &tag, 1, PRTE_RML_TAG); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&data); return rc; } /* copy the payload into the new buffer - this is non-destructive, so our * caller is still responsible for releasing any memory in the buffer they * gave to us */ rc = PMIx_Data_copy_payload(&data, message); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&data); return rc; } /* see if we want to compress this message */ if (PMIx_Data_compress((uint8_t *) data.base_ptr, data.bytes_used, (uint8_t **) &bo.bytes, &sz)) { /* the data was compressed - mark that we compressed it */ compressed = true; bo.size = sz; } else { /* mark that it was not compressed */ compressed = false; bo.bytes = data.base_ptr; bo.size = data.bytes_used; data.base_ptr = NULL; data.bytes_used = 0; } PMIX_DATA_BUFFER_DESTRUCT(&data); rc = PMIx_Data_pack(NULL, buffer, &compressed, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_BYTE_OBJECT_DESTRUCT(&bo); return rc; } rc = PMIx_Data_pack(NULL, buffer, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_BYTE_OBJECT_DESTRUCT(&bo); return rc; } PMIX_BYTE_OBJECT_DESTRUCT(&bo); return PRTE_SUCCESS; } int prte_pack_ctrl_options(pmix_byte_object_t *ctrlsbo, const pmix_info_t *info, size_t ninfo) { pmix_data_buffer_t ctrlbuf; pmix_status_t rc; PMIx_Data_buffer_construct(&ctrlbuf); rc = PMIx_Data_pack(NULL, &ctrlbuf, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIx_Data_buffer_destruct(&ctrlbuf); return rc; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, &ctrlbuf, (void*)info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIx_Data_buffer_destruct(&ctrlbuf); return rc; } } /* even if the control buffer is empty, we still have * to pack the byte object for it to ensure proper * unpacking on the remote end */ rc = PMIx_Data_unload(&ctrlbuf, ctrlsbo); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIx_Data_buffer_destruct(&ctrlbuf); return rc; } PMIx_Data_buffer_destruct(&ctrlbuf); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/grpcomm/base/Makefile.am0000664000175000017500000000207115203542132021600 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # headers += \ base/base.h libprtemca_grpcomm_la_SOURCES += \ base/grpcomm_base_select.c \ base/grpcomm_base_frame.c prrte-4.1.1~rc2/src/mca/grpcomm/base/grpcomm_base_frame.c0000664000175000017500000000636715203542132023534 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/mca.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_output.h" /* * The following file was created by configure. It contains extern * statements and the definition of an array of pointers to each * component's public pmix_mca_base_component_t struct. */ #include "src/mca/grpcomm/base/static-components.h" /* * Global variables */ prte_grpcomm_base_t prte_grpcomm_base = { .context_id = UINT32_MAX }; prte_grpcomm_base_module_t prte_grpcomm = {0}; static int prte_grpcomm_base_close(void) { if (NULL != prte_grpcomm.finalize) { prte_grpcomm.finalize(); } return pmix_mca_base_framework_components_close(&prte_grpcomm_base_framework, NULL); } /** * Function for finding and opening either all MCA components, or the one * that was specifically requested via a MCA parameter. */ static int prte_grpcomm_base_open(pmix_mca_base_open_flag_t flags) { return pmix_mca_base_framework_components_open(&prte_grpcomm_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, grpcomm, "GRPCOMM", NULL, prte_grpcomm_base_open, prte_grpcomm_base_close, prte_grpcomm_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); static void grpcon(prte_pmix_grp_caddy_t *p) { PMIX_CONSTRUCT_LOCK(&p->lock); p->op = PMIX_GROUP_NONE; p->grpid = NULL; p->procs = NULL; p->nprocs = 0; p->directives = NULL; p->ndirs = 0; p->info = NULL; p->ninfo = 0; p->cbfunc = NULL; p->cbdata = NULL; } static void grpdes(prte_pmix_grp_caddy_t *p) { PMIX_DESTRUCT_LOCK(&p->lock); if (NULL != p->grpid) { free(p->grpid); } if (NULL != p->info) { PMIX_INFO_FREE(p->info, p->ninfo); } } PMIX_CLASS_INSTANCE(prte_pmix_grp_caddy_t, pmix_object_t, grpcon, grpdes); prrte-4.1.1~rc2/src/mca/grpcomm/base/grpcomm_base_select.c0000664000175000017500000000411515203542132023706 0ustar alastairalastair/* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/grpcomm/base/base.h" /** * Function for selecting one component from all those that are * available. */ int prte_grpcomm_base_select(void) { prte_grpcomm_base_component_t *best_component = NULL; prte_grpcomm_base_module_t *best_module = NULL; pmix_status_t rc; /* * Select the best component */ rc = pmix_mca_base_select("grpcomm", prte_grpcomm_base_framework.framework_output, &prte_grpcomm_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL); if (PMIX_SUCCESS != rc) { /* This will only happen if no component was selected */ return PRTE_ERR_NOT_FOUND; } /* Save the winner */ prte_grpcomm = *best_module; /* give it a chance to initialize */ if (NULL != prte_grpcomm.init) { prte_grpcomm.init(); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/grpcomm/Makefile.am0000664000175000017500000000236215203542132020671 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_grpcomm.la libprtemca_grpcomm_la_SOURCES = # local files headers = grpcomm.h libprtemca_grpcomm_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/grpcomm/direct/0000775000175000017500000000000015203542132020104 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct.h0000664000175000017500000002067215203542132023262 0ustar alastairalastair/* -*- C -*- * * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #ifndef GRPCOMM_DIRECT_H #define GRPCOMM_DIRECT_H #include "prte_config.h" #include "src/mca/grpcomm/grpcomm.h" BEGIN_C_DECLS /* * Grpcomm interfaces */ typedef struct { prte_grpcomm_base_component_t super; // track ongoing fence operations - list of prte_grpcomm_fence_t pmix_list_t fence_ops; // track ongoiong group operations - list of prte_grpcomm_group_t pmix_list_t group_ops; } prte_grpcomm_direct_component_t; PRTE_MODULE_EXPORT extern prte_grpcomm_direct_component_t prte_mca_grpcomm_direct_component; extern prte_grpcomm_base_module_t prte_grpcomm_direct_module; /* Define collective signatures so we don't need to * track global collective id's. We provide a unique * signature struct for each collective type so that * they can be customized for that collective without * interfering with other collectives */ typedef struct { pmix_object_t super; pmix_proc_t *signature; size_t sz; } prte_grpcomm_direct_fence_signature_t; PRTE_MODULE_EXPORT PMIX_CLASS_DECLARATION(prte_grpcomm_direct_fence_signature_t); typedef struct { pmix_object_t super; pmix_group_operation_t op; char *groupID; bool assignID; size_t ctxid; bool ctxid_assigned; pmix_proc_t *members; // initially supplied procs size_t nmembers; size_t bootstrap; bool follower; pmix_proc_t *addmembers; // procs supplied as add-members size_t naddmembers; pmix_proc_t *final_order; size_t nfinal; } prte_grpcomm_direct_group_signature_t; PRTE_MODULE_EXPORT PMIX_CLASS_DECLARATION(prte_grpcomm_direct_group_signature_t); /* Internal component object for tracking ongoing * allgather operations */ typedef struct { pmix_list_item_t super; /* collective's signature */ prte_grpcomm_direct_fence_signature_t *sig; pmix_status_t status; /* collection bucket */ pmix_data_buffer_t bucket; /* participating daemons */ pmix_rank_t *dmns; /** number of participating daemons */ size_t ndmns; /** my index in the dmns array */ unsigned long my_rank; /* number of buckets expected */ size_t nexpected; /* number reported in */ size_t nreported; /* controls values */ int timeout; /* callback function */ pmix_modex_cbfunc_t cbfunc; /* user-provided callback data */ void *cbdata; } prte_grpcomm_fence_t; PMIX_CLASS_DECLARATION(prte_grpcomm_fence_t); /* Internal component object for tracking ongoing * group operations */ typedef struct { pmix_list_item_t super; /* collective's signature */ prte_grpcomm_direct_group_signature_t *sig; pmix_status_t status; /* participating daemons */ pmix_rank_t *dmns; /** number of participating daemons */ size_t ndmns; /** my index in the dmns array */ unsigned long my_rank; /* type of collective */ bool bootstrap; /*** NON-BOOTSTRAP TRACKERS ***/ size_t nexpected; // number of buckets expected size_t nreported; // number reported in /*** BOOTSTRAP TRACKERS ***/ // "leaders" are group members reporting as // themselves for bootstrap - they know how // many leaders there are (which is in the bootstrap // parameter), but not who they are. Bootstrap is // complete when nleaders_reported == bootstrap // AND naddmembers_reported == naddmembers size_t nleaders; // number of leaders expected size_t nleaders_reported; // number reported in // "add-members" are procs that report with NULL // for the proc parameter - thereby indicating that // they don't know the other procs in the group size_t nfollowers; // number of add-member procs expected to participate size_t nfollowers_reported; // number reported in /* controls values */ bool assignID; int timeout; size_t memsize; void *grpinfo; // info list of group info void *endpts; // info list of endpts /* callback function */ pmix_info_cbfunc_t cbfunc; /* user-provided callback data */ void *cbdata; } prte_grpcomm_group_t; PMIX_CLASS_DECLARATION(prte_grpcomm_group_t); typedef struct { pmix_object_t super; prte_event_t ev; prte_grpcomm_direct_fence_signature_t *sig; pmix_data_buffer_t *buf; pmix_proc_t *procs; size_t nprocs; pmix_info_t *info; size_t ninfo; char *data; size_t ndata; pmix_modex_cbfunc_t cbfunc; void *cbdata; } prte_pmix_fence_caddy_t; PMIX_CLASS_DECLARATION(prte_pmix_fence_caddy_t); /* xcast functions */ PRTE_MODULE_EXPORT extern int prte_grpcomm_direct_xcast(prte_rml_tag_t tag, pmix_data_buffer_t *msg); PRTE_MODULE_EXPORT extern void prte_grpcomm_direct_xcast_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tg, void *cbdata); /* fence functions */ PRTE_MODULE_EXPORT extern int prte_grpcomm_direct_fence(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata); PRTE_MODULE_EXPORT extern void prte_grpcomm_direct_fence_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); PRTE_MODULE_EXPORT extern void prte_grpcomm_direct_fence_release(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); /* group functions */ PRTE_MODULE_EXPORT extern int prte_grpcomm_direct_group(pmix_group_operation_t op, char *grpid, const pmix_proc_t procs[], size_t nprocs, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata); PRTE_MODULE_EXPORT extern void prte_grpcomm_direct_grp_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); PRTE_MODULE_EXPORT extern void prte_grpcomm_direct_grp_release(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata); static inline void print_signature(prte_grpcomm_direct_group_signature_t *sig) { char **msg = NULL; char *tmp; size_t n; PMIx_Argv_append_nosize(&msg, "SIGNATURE:"); pmix_asprintf(&tmp, "\tOP: %s", PMIx_Group_operation_string(sig->op)); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); pmix_asprintf(&tmp, "\tGRPID: %s", sig->groupID); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); pmix_asprintf(&tmp, "\tASSIGN CTXID: %s", sig->assignID ? "T" : "F"); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); if (sig->assignID) { pmix_asprintf(&tmp, "\tCTXID: %lu", sig->ctxid); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); } pmix_asprintf(&tmp, "\tNMEMBERS: %lu", sig->nmembers); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); if (0 < sig->nmembers) { for (n=0; n < sig->nmembers; n++) { pmix_asprintf(&tmp, "\t\t%s", PMIX_NAME_PRINT(&sig->members[n])); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); } } pmix_asprintf(&tmp, "\tBOOTSTRAP: %lu", sig->bootstrap); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); pmix_asprintf(&tmp, "\tFOLLOWER: %s", sig->follower ? "T" : "F"); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); pmix_asprintf(&tmp, "\tNADDMEMBERS: %lu", sig->naddmembers); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); if (0 < sig->naddmembers) { for (n=0; n < sig->naddmembers; n++) { pmix_asprintf(&tmp, "\t\t%s", PMIX_NAME_PRINT(&sig->addmembers[n])); PMIx_Argv_append_nosize(&msg, tmp); free(tmp); } } tmp = PMIx_Argv_join(msg, '\n'); PMIx_Argv_free(msg); pmix_output(0, "%s", tmp); free(tmp); } END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/grpcomm/direct/owner.txt0000664000175000017500000000024415203542132021777 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct_group.c0000664000175000017500000017052515203542132024474 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; -*- */ /* * Copyright (c) 2007 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All * rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/util/name_fns.h" #include "src/util/nidmap.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "grpcomm_direct.h" #include "src/mca/grpcomm/base/base.h" static void group(int sd, short args, void *cbdata); static prte_grpcomm_group_t *get_tracker(prte_grpcomm_direct_group_signature_t *sig, bool create); static int create_dmns(prte_grpcomm_direct_group_signature_t *sig, pmix_rank_t **dmns, size_t *ndmns); static int pack_signature(pmix_data_buffer_t *buf, prte_grpcomm_direct_group_signature_t *sig); static int unpack_signature(pmix_data_buffer_t *buf, prte_grpcomm_direct_group_signature_t **sig); int prte_grpcomm_direct_group(pmix_group_operation_t op, char *grpid, const pmix_proc_t procs[], size_t nprocs, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata) { prte_pmix_grp_caddy_t *cd; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group %s for \"%s\" with %lu procs", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PMIx_Group_operation_string(op), grpid, nprocs)); cd = PMIX_NEW(prte_pmix_grp_caddy_t); cd->op = op; cd->grpid = strdup(grpid); cd->procs = procs; cd->nprocs = nprocs; cd->directives = directives; cd->ndirs = ndirs; cd->cbfunc = cbfunc; cd->cbdata = cbdata; /* must push this into the event library to ensure we can * access framework-global data safely */ prte_event_set(prte_event_base, &cd->ev, -1, PRTE_EV_WRITE, group, cd); PMIX_POST_OBJECT(cd); prte_event_active(&cd->ev, PRTE_EV_WRITE, 1); return PRTE_SUCCESS; } static void group(int sd, short args, void *cbdata) { prte_pmix_grp_caddy_t *cd = (prte_pmix_grp_caddy_t*)cbdata; prte_grpcomm_direct_group_signature_t sig; prte_grpcomm_group_t *coll; size_t i; pmix_data_buffer_t *relay; pmix_status_t rc, st = PMIX_SUCCESS; int timeout = 0; void *endpts, *grpinfo; pmix_data_array_t darray; pmix_info_t *info; size_t ninfo; PRTE_HIDE_UNUSED_PARAMS(sd, args); /* compute the signature of this collective */ PMIX_CONSTRUCT(&sig, prte_grpcomm_direct_group_signature_t); sig.groupID = strdup(cd->grpid); if (NULL != cd->procs) { sig.nmembers = cd->nprocs; PMIX_PROC_CREATE(sig.members, sig.nmembers); memcpy(sig.members, cd->procs, sig.nmembers * sizeof(pmix_proc_t)); } else { // if no procs were given, then this must be a follower sig.follower = true; } sig.op = cd->op; // setup to track endpts and grpinfo endpts = PMIx_Info_list_start(); grpinfo = PMIx_Info_list_start(); /* check the directives */ for (i = 0; i < cd->ndirs; i++) { /* see if they want a context id assigned */ if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_GROUP_ASSIGN_CONTEXT_ID)) { sig.assignID = PMIX_INFO_TRUE(&cd->directives[i]); } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_GROUP_BOOTSTRAP)) { PMIX_VALUE_GET_NUMBER(rc, &cd->directives[i].value, sig.bootstrap, size_t); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&sig); } } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_LOCAL_COLLECTIVE_STATUS)) { PMIX_VALUE_GET_NUMBER(rc, &cd->directives[i].value, st, pmix_status_t); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&sig); goto error; } } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_TIMEOUT)) { PMIX_VALUE_GET_NUMBER(rc, &cd->directives[i].value, timeout, int); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&sig); goto error; } } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_GROUP_ADD_MEMBERS)) { // there is only one of these as it is aggregated by the // PMIx server library sig.addmembers = (pmix_proc_t*)cd->directives[i].value.data.darray->array; sig.naddmembers = cd->directives[i].value.data.darray->size; } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_GROUP_INFO)) { rc = PMIx_Info_list_xfer(grpinfo, &cd->directives[i]); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_PROC_DATA)) { rc = PMIx_Info_list_xfer(endpts, &cd->directives[i]); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } } else if (PMIX_CHECK_KEY(&cd->directives[i], PMIX_GROUP_FINAL_MEMBERSHIP_ORDER)) { sig.final_order = (pmix_proc_t*)cd->directives[i].value.data.darray->array; sig.nfinal = cd->directives[i].value.data.darray->size; } } /* create a tracker for this operation */ if (NULL == (coll = get_tracker(&sig, true))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PMIX_RELEASE(cd); PMIX_DESTRUCT(&sig); return; } coll->cbfunc = cd->cbfunc; coll->cbdata = cd->cbdata; // create the relay buffer PMIX_DATA_BUFFER_CREATE(relay); /* pack the signature */ rc = pack_signature(relay, &sig); // protect the inbound directives sig.addmembers = NULL; sig.naddmembers = 0; if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } // pack the local collective status rc = PMIx_Data_pack(NULL, relay, &st, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } // pack any timeout directive rc = PMIx_Data_pack(NULL, relay, &timeout, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } if (PMIX_GROUP_CONSTRUCT == sig.op) { // pack any group info PMIx_Info_list_convert(grpinfo, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, relay, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, relay, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); // pack any endpts PMIx_Info_list_convert(endpts, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, relay, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, relay, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_DESTRUCT(&sig); goto error; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); } PMIx_Info_list_release(grpinfo); PMIx_Info_list_release(endpts); /* if this is a bootstrap operation, send it directly to the HNP */ if (coll->bootstrap) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:grp bootstrap sending %lu bytes to HNP", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), relay->bytes_used)); PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, relay, PRTE_RML_TAG_GROUP); if (PRTE_SUCCESS != rc) { PMIX_RELEASE(relay); rc = prte_pmix_convert_rc(rc); PMIX_DESTRUCT(&sig); goto error; } PMIX_DESTRUCT(&sig); return; } PMIX_DESTRUCT(&sig); /* send this to ourselves for processing */ PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:grp sending to ourself", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_RML_SEND(rc, PRTE_PROC_MY_NAME->rank, relay, PRTE_RML_TAG_GROUP); if (PRTE_SUCCESS != rc) { PMIX_RELEASE(relay); rc = prte_pmix_convert_rc(rc); goto error; } return; error: if (NULL != cd->cbfunc) { cd->cbfunc(rc, NULL, 0, cd->cbdata, NULL, NULL); } PMIX_RELEASE(cd); } void prte_grpcomm_direct_grp_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { int32_t cnt; int rc, timeout; size_t m, n, ninfo, nfinal = 0, nendpts, ngrpinfo; pmix_proc_t *finalmembership = NULL; bool found; pmix_list_t nmlist; prte_namelist_t *nm; pmix_data_array_t darray; pmix_status_t st; pmix_info_t *info = NULL, *endpts, *grpinfo = NULL; prte_grpcomm_direct_group_signature_t *sig = NULL; pmix_data_buffer_t *reply; prte_grpcomm_group_t *coll; PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata); PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct grp recvd from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender))); // empty buffer indicates lost connection if (NULL == buffer || NULL == buffer->unpack_ptr) { PMIX_ERROR_LOG(PMIX_ERR_EMPTY); return; } /* unpack the signature */ rc = unpack_signature(buffer, &sig); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* check for the tracker and create it if not found */ if (NULL == (coll = get_tracker(sig, true))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PMIX_RELEASE(sig); return; } // unpack the local collective status cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &st, &cnt, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(sig); return; } if (PMIX_SUCCESS != st && PMIX_SUCCESS == coll->status) { coll->status = st; } // unpack any timeout directive cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &timeout, &cnt, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(sig); return; } if (coll->timeout < timeout) { coll->timeout = timeout; } if (PMIX_GROUP_CONSTRUCT == sig->op) { /* unpack the number of group infos */ cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &ngrpinfo, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(sig); return; } if (0 < ngrpinfo) { PMIX_INFO_CREATE(grpinfo, ngrpinfo); cnt = ngrpinfo; rc = PMIx_Data_unpack(NULL, buffer, grpinfo, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(grpinfo, ngrpinfo); PMIX_RELEASE(sig); return; } for (n=0; n < ngrpinfo; n++) { rc = PMIx_Info_list_xfer(coll->grpinfo, &grpinfo[n]); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } } PMIX_INFO_FREE(grpinfo, ngrpinfo); } /* unpack the number of endpoints */ cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &nendpts, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); if (NULL != grpinfo) { PMIX_INFO_FREE(grpinfo, ngrpinfo); } PMIX_RELEASE(sig); return; } if (0 < nendpts) { PMIX_INFO_CREATE(endpts, nendpts); cnt = nendpts; rc = PMIx_Data_unpack(NULL, buffer, endpts, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); if (NULL != grpinfo) { PMIX_INFO_FREE(grpinfo, ngrpinfo); } PMIX_INFO_FREE(endpts, nendpts); PMIX_RELEASE(sig); return; } for (n=0; n < nendpts; n++) { rc = PMIx_Info_list_xfer(coll->endpts, &endpts[n]); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } } PMIX_INFO_FREE(endpts, nendpts); } } /* track procs reported for collective */ if (0 < sig->bootstrap) { // this came from a bootstrap leader coll->nleaders_reported++; pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct group recv leader nrep %d of %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) coll->nleaders_reported, (int) coll->nleaders); } else if (sig->follower) { // came from a bootstrap follower coll->nfollowers_reported++; pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct group recv follower nrep %d of %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) coll->nfollowers_reported, (int) coll->nfollowers); } else { // group collective op coll->nreported++; pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct group recv nexpected %d nrep %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) coll->nexpected, (int) coll->nreported); } /* see if everyone has reported */ if ((coll->bootstrap && (coll->nleaders_reported == coll->nleaders && coll->nfollowers_reported == coll->nfollowers)) || (!coll->bootstrap && coll->nreported == coll->nexpected)) { if (PRTE_PROC_IS_MASTER) { pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct group HNP reports complete for %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), coll->sig->groupID); /* the allgather is complete - send the xcast */ if (PMIX_GROUP_CONSTRUCT == sig->op) { /* if we were asked to provide a context id, do so */ if (coll->sig->assignID) { coll->sig->ctxid = prte_grpcomm_base.context_id; --prte_grpcomm_base.context_id; coll->sig->ctxid_assigned = true; } // construct the final membership PMIX_CONSTRUCT(&nmlist, pmix_list_t); // sadly, an exhaustive search for (m=0; m < coll->sig->nmembers; m++) { found = false; PMIX_LIST_FOREACH(nm, &nmlist, prte_namelist_t) { if (PMIX_CHECK_PROCID(&coll->sig->members[m], &nm->name)) { // if the new one is rank=WILDCARD, then ensure // we keep it as wildcard if (PMIX_RANK_WILDCARD == coll->sig->members[m].rank) { nm->name.rank = PMIX_RANK_WILDCARD; } found = true; break; } } if (!found) { nm = PMIX_NEW(prte_namelist_t); memcpy(&nm->name, &coll->sig->members[m], sizeof(pmix_proc_t)); pmix_list_append(&nmlist, &nm->super); } } // now check any added members for (m=0; m < coll->sig->naddmembers; m++) { found = false; PMIX_LIST_FOREACH(nm, &nmlist, prte_namelist_t) { if (PMIX_CHECK_PROCID(&coll->sig->addmembers[m], &nm->name)) { // if the new one is rank=WILDCARD, then ensure // we keep it as wildcard if (PMIX_RANK_WILDCARD == coll->sig->addmembers[m].rank) { nm->name.rank = PMIX_RANK_WILDCARD; } found = true; break; } } if (!found) { nm = PMIX_NEW(prte_namelist_t); memcpy(&nm->name, &coll->sig->addmembers[m], sizeof(pmix_proc_t)); pmix_list_append(&nmlist, &nm->super); } } // create the full membership array nfinal = pmix_list_get_size(&nmlist); PMIX_PROC_CREATE(finalmembership, nfinal); m = 0; PMIX_LIST_FOREACH(nm, &nmlist, prte_namelist_t) { memcpy(&finalmembership[m], &nm->name, sizeof(pmix_proc_t)); ++m; } PMIX_LIST_DESTRUCT(&nmlist); // if they gave us a final order, then sort the final membership // accordingly. Note that order entries that consist of nspace,wildcard // indicate that all participants from the given nspace should be // included in the final membership at that point - it does NOT mean // that all procs from that nspace are included in the final membership if (NULL != coll->sig->final_order) { PMIX_CONSTRUCT(&nmlist, pmix_list_t); for (m=0; m < coll->sig->nfinal; m++) { // search the array of final members to capture those that match for (n=0; n < nfinal; n++) { if (PMIX_CHECK_PROCID(&coll->sig->final_order[m], &finalmembership[n])) { // add this proc to the final list nm = PMIX_NEW(prte_namelist_t); memcpy(&nm->name, &finalmembership[n], sizeof(pmix_proc_t)); pmix_list_append(&nmlist, &nm->super); // the final order may have included rank=wildcard - if so, // then we have to continue if (PMIX_RANK_WILDCARD != coll->sig->final_order[m].rank) { // nope - can only match once break; } } } } // did we lose anyone? if (nfinal != pmix_list_get_size(&nmlist)) { pmix_show_help("help-prte-runtime.txt", "bad-final-order", true); coll->status = PMIX_ERR_BAD_PARAM; goto answer; } // just overwrite the final array m = 0; PMIX_LIST_FOREACH(nm, &nmlist, prte_namelist_t) { memcpy(&finalmembership[m], &nm->name, sizeof(pmix_proc_t)); ++m; } PMIX_LIST_DESTRUCT(&nmlist); // zero out the final order cache - no need to send it around PMIX_PROC_FREE(coll->sig->final_order, coll->sig->nfinal); coll->sig->final_order = NULL; coll->sig->nfinal = 0; } else { /* sort the procs so everyone gets the same order */ qsort(finalmembership, nfinal, sizeof(pmix_proc_t), pmix_util_compare_proc); } } answer: // CONSTRUCT THE RELEASE MESSAGE PMIX_DATA_BUFFER_CREATE(reply); /* pack the signature */ rc = pack_signature(reply, coll->sig); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); PMIX_PROC_FREE(finalmembership, nfinal); return; } /* pack the status */ rc = PMIx_Data_pack(NULL, reply, &coll->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); PMIX_PROC_FREE(finalmembership, nfinal); return; } if (PMIX_GROUP_CONSTRUCT == sig->op) { // pack the final membership rc = PMIx_Data_pack(NULL, reply, &nfinal, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); PMIX_PROC_FREE(finalmembership, nfinal); return; } if (0 < nfinal) { rc = PMIx_Data_pack(NULL, reply, finalmembership, nfinal, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } PMIX_PROC_FREE(finalmembership, nfinal); } // pack any group info PMIx_Info_list_convert(coll->grpinfo, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, reply, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, reply, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); // pack any endpts PMIx_Info_list_convert(coll->endpts, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, reply, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, reply, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); } /* send the release via xcast */ (void) prte_grpcomm.xcast(PRTE_RML_TAG_GROUP_RELEASE, reply); } else { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct allgather rollup complete - sending to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(PRTE_PROC_MY_PARENT))); // setup to relay our rollup results PMIX_DATA_BUFFER_CREATE(reply); /* pack the signature */ rc = pack_signature(reply, coll->sig); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } // pack the local collective status rc = PMIx_Data_pack(NULL, reply, &coll->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } // pack any timeout directive rc = PMIx_Data_pack(NULL, reply, &coll->timeout, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } if (PMIX_GROUP_CONSTRUCT == sig->op) { // pack any group info PMIx_Info_list_convert(coll->grpinfo, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, reply, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, reply, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); // pack any endpts PMIx_Info_list_convert(coll->endpts, &darray); info = (pmix_info_t*)darray.array; ninfo = darray.size; rc = PMIx_Data_pack(NULL, reply, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } if (0 < ninfo) { rc =PMIx_Data_pack(NULL, reply, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } } PMIX_DATA_ARRAY_DESTRUCT(&darray); } /* send the info to our parent */ PRTE_RML_SEND(rc, PRTE_PROC_MY_PARENT->rank, reply, PRTE_RML_TAG_GROUP); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(sig); return; } } } PMIX_RELEASE(sig); } static void relcb(void *cbdata) { prte_pmix_grp_caddy_t *cd = (prte_pmix_grp_caddy_t*)cbdata; PMIX_RELEASE(cd); } static void lkopcbfunc(int status, void *cbdata) { prte_pmix_grp_caddy_t *cd = (prte_pmix_grp_caddy_t*)cbdata; PRTE_HIDE_UNUSED_PARAMS(status); cd->lock.status = status; PMIX_WAKEUP_THREAD(&cd->lock); } static void find_delete_tracker(prte_grpcomm_direct_group_signature_t *sig) { prte_grpcomm_group_t *coll; PMIX_LIST_FOREACH(coll, &prte_mca_grpcomm_direct_component.group_ops, prte_grpcomm_group_t) { // must match groupID's if (0 == strcmp(sig->groupID, coll->sig->groupID)) { pmix_list_remove_item(&prte_mca_grpcomm_direct_component.group_ops, &coll->super); PMIX_RELEASE(coll); return; } } } void prte_grpcomm_direct_grp_release(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { prte_grpcomm_group_t *coll; prte_grpcomm_direct_group_signature_t *sig = NULL; prte_pmix_grp_caddy_t cd2, *cd; int32_t cnt; pmix_status_t rc = PMIX_SUCCESS, st = PMIX_SUCCESS; pmix_proc_t *finalmembership = NULL; size_t nfinal = 0; size_t nendpts = 0; size_t ngrpinfo = 0; size_t n; pmix_data_array_t darray; pmix_info_t *grpinfo = NULL; pmix_info_t *endpts = NULL; prte_pmix_server_pset_t *pset; void *ilist, *nlist; PRTE_HIDE_UNUSED_PARAMS(status, sender, tag, cbdata); PMIX_ACQUIRE_OBJECT(cd); pmix_output_verbose(2, prte_pmix_server_globals.output, "%s group release recvd", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); // unpack the signature rc = unpack_signature(buffer, &sig); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); return; } /* check for the tracker - okay if not found, it just * means that we had no local participants */ coll = get_tracker(sig, false); // unpack the status cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &st, &cnt, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; } /* if this was a destruct operation, then there is nothing * further to unpack */ if (PMIX_GROUP_DESTRUCT == sig->op) { /* find this group ID on our list of groups */ PMIX_LIST_FOREACH(pset, &prte_pmix_server_globals.groups, prte_pmix_server_pset_t) { if (0 == strcmp(pset->name, sig->groupID)) { pmix_list_remove_item(&prte_pmix_server_globals.groups, &pset->super); PMIX_RELEASE(pset); break; } } if (NULL != coll && NULL != coll->cbfunc) { /* return to the local procs in the collective */ coll->cbfunc(st, NULL, 0, coll->cbdata, NULL, NULL); } // remove the tracker, if found find_delete_tracker(sig); PMIX_RELEASE(sig); return; } // setup to cache info ilist = PMIx_Info_list_start(); nlist = PMIx_Info_list_start(); // must be a construct operation - continue unpacking if (PMIX_SUCCESS != st) { PMIX_INFO_LIST_RELEASE(ilist); goto notify; } if (sig->ctxid_assigned) { PMIX_INFO_LIST_ADD(rc, ilist, PMIX_GROUP_CONTEXT_ID, &sig->ctxid, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } PMIX_INFO_LIST_ADD(rc, nlist, PMIX_GROUP_CONTEXT_ID, &sig->ctxid, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } } // unpack the final membership cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &nfinal, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } if (0 < nfinal) { PMIX_PROC_CREATE(finalmembership, nfinal); cnt = nfinal; rc = PMIx_Data_unpack(NULL, buffer, finalmembership, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } // pass back the final group membership darray.type = PMIX_PROC; darray.array = finalmembership; darray.size = nfinal; // load the array - note: this copies the array! PMIX_INFO_LIST_ADD(rc, nlist, PMIX_GROUP_MEMBERSHIP, &darray, PMIX_DATA_ARRAY); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } } // unpack group info cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &ngrpinfo, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } if (0 < ngrpinfo) { PMIX_INFO_CREATE(grpinfo, ngrpinfo); cnt = ngrpinfo; rc = PMIx_Data_unpack(NULL, buffer, grpinfo, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(grpinfo, ngrpinfo); goto notify; } // transfer them to both lists for (n=0; n < ngrpinfo; n++) { rc = PMIx_Info_list_add_value(ilist, PMIX_GROUP_INFO, &grpinfo[n].value); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(grpinfo, ngrpinfo); goto notify; } rc = PMIx_Info_list_add_value(nlist, PMIX_GROUP_INFO, &grpinfo[n].value); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(grpinfo, ngrpinfo); goto notify; } } PMIX_INFO_FREE(grpinfo, ngrpinfo); } // unpack endpts cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &nendpts, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); goto notify; } if (0 < nendpts) { PMIX_INFO_CREATE(endpts, nendpts); cnt = nendpts; rc = PMIx_Data_unpack(NULL, buffer, endpts, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(endpts, nendpts); goto notify; } // transfer them to both lists for (n=0; n < nendpts; n++) { rc = PMIx_Info_list_add_value(ilist, PMIX_GROUP_ENDPT_DATA, &endpts[n].value); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(endpts, nendpts); goto notify; } rc = PMIx_Info_list_add_value(nlist, PMIX_GROUP_ENDPT_DATA, &endpts[n].value); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); st = rc; PMIX_INFO_LIST_RELEASE(ilist); PMIX_INFO_FREE(endpts, nendpts); goto notify; } } PMIX_INFO_FREE(endpts, nendpts); } // PRRTE automatically ensures that all daemons register all jobs // with their local PMIx server, regardless of whether or not // that daemon hosts any local clients of that job. So we // do not need to collect/pass job data for participants // in the group construct // pass the information down to the PMIx server PMIX_INFO_LIST_CONVERT(rc, ilist, &darray); PMIX_CONSTRUCT(&cd2, prte_pmix_grp_caddy_t); cd2.info = (pmix_info_t*)darray.array; cd2.ninfo = darray.size; PMIX_INFO_LIST_RELEASE(ilist); rc = PMIx_server_register_resources(cd2.info, cd2.ninfo, lkopcbfunc, &cd2); if (PMIX_SUCCESS == rc) { PMIX_WAIT_THREAD(&cd2.lock); rc = cd2.lock.status; } PMIX_DESTRUCT(&cd2); if (PMIX_SUCCESS == st) { /* add it to our list of known groups */ pset = PMIX_NEW(prte_pmix_server_pset_t); pset->name = strdup(sig->groupID); if (NULL != finalmembership) { pset->num_members = nfinal; PMIX_PROC_CREATE(pset->members, pset->num_members); memcpy(pset->members, finalmembership, nfinal * sizeof(pmix_proc_t)); } pmix_list_append(&prte_pmix_server_globals.groups, &pset->super); } notify: // regardless of prior error, we MUST notify any pending clients // so they don't hang if (NULL != coll && NULL != coll->cbfunc) { // service the procs that are part of the collective // convert for returning to PMIx server library cd = PMIX_NEW(prte_pmix_grp_caddy_t); if (PMIX_SUCCESS == st) { PMIX_INFO_LIST_CONVERT(rc, nlist, &darray); if (PMIX_SUCCESS != rc && PMIX_ERR_EMPTY != rc) { PMIX_ERROR_LOG(rc); } cd->info = (pmix_info_t*)darray.array; cd->ninfo = darray.size; } /* return to the PMIx server library for relay to * local procs in the operation */ coll->cbfunc(st, cd->info, cd->ninfo, coll->cbdata, relcb, (void*)cd); } if (0 < nendpts) { PMIX_INFO_FREE(endpts, nendpts); } if (0 < ngrpinfo) { PMIX_INFO_FREE(grpinfo, ngrpinfo); } if (0 < nfinal) { PMIX_PROC_FREE(finalmembership, nfinal); } PMIX_INFO_LIST_RELEASE(nlist); // remove this collective from our tracker find_delete_tracker(sig); PMIX_RELEASE(sig); } static prte_grpcomm_group_t *get_tracker(prte_grpcomm_direct_group_signature_t *sig, bool create) { prte_grpcomm_group_t *coll; int rc; pmix_proc_t *p; size_t n, nmb; pmix_list_t plist; prte_namelist_t *nm; bool found; if (NULL == sig->groupID) { return NULL; } /* search the existing tracker list to see if this already exists - we * default to using the groupID if one is given, otherwise we fallback * to the array of participating procs */ PMIX_LIST_FOREACH(coll, &prte_mca_grpcomm_direct_component.group_ops, prte_grpcomm_group_t) { // must match groupID's and ops if (0 == strcmp(sig->groupID, coll->sig->groupID) && sig->op == coll->sig->op) { pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group:returning existing collective %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), sig->groupID); // if this is a bootstrap, then we have to track the number of leaders if (0 < sig->bootstrap) { if (0 < coll->nleaders) { if (coll->nleaders != sig->bootstrap) { // this is an error PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return NULL; } } else { // collective tracker could have been created by a follower, // which means nleaders will not have been set coll->nleaders = sig->bootstrap; } coll->bootstrap = true; // add this proc to the list of members PMIX_CONSTRUCT(&plist, pmix_list_t); for (n=0; n < sig->nmembers; n++) { // see if we already have this proc found = false; for (nmb=0; nmb < coll->sig->nmembers; nmb++) { if (PMIX_CHECK_PROCID(&sig->members[n], &coll->sig->members[nmb])) { // yes, we do found = true; // check for wildcard as that needs to be retained if (PMIX_RANK_WILDCARD == sig->members[n].rank) { coll->sig->members[n].rank = PMIX_RANK_WILDCARD; } break; } } if (!found) { // cache the proc nm = PMIX_NEW(prte_namelist_t); memcpy(&nm->name, &sig->members[n], sizeof(pmix_proc_t)); pmix_list_append(&plist, &nm->super); } } // add any missing procs to the addmembers if (0 < pmix_list_get_size(&plist)) { n = coll->sig->nmembers + pmix_list_get_size(&plist); PMIX_PROC_CREATE(p, n); if (NULL != coll->sig->members) { memcpy(p, coll->sig->members, coll->sig->nmembers * sizeof(pmix_proc_t)); } n = coll->sig->nmembers; PMIX_LIST_FOREACH(nm, &plist, prte_namelist_t) { memcpy(&p[n], &nm->name, sizeof(pmix_proc_t)); ++n; } PMIX_LIST_DESTRUCT(&plist); if (NULL != coll->sig->members) { PMIX_PROC_FREE(coll->sig->members, coll->sig->nmembers); } coll->sig->members = p; coll->sig->nmembers = n; } } else if (sig->follower) { // just ensure the bootstrap flag is set coll->bootstrap = true; } // if we are adding members, aggregate them if (0 < sig->naddmembers) { PMIX_CONSTRUCT(&plist, pmix_list_t); for (n=0; n < sig->naddmembers; n++) { // see if we already have this proc found = false; for (nmb=0; nmb < coll->sig->naddmembers; nmb++) { if (PMIX_CHECK_PROCID(&sig->addmembers[n], &coll->sig->addmembers[nmb])) { // yes, we do found = true; // check for wildcard as that needs to be retained if (PMIX_RANK_WILDCARD == sig->addmembers[n].rank) { coll->sig->addmembers[n].rank = PMIX_RANK_WILDCARD; } break; } } if (!found) { // cache the proc nm = PMIX_NEW(prte_namelist_t); memcpy(&nm->name, &sig->addmembers[n], sizeof(pmix_proc_t)); pmix_list_append(&plist, &nm->super); } } // add any missing procs to the addmembers if (0 < pmix_list_get_size(&plist)) { n = coll->sig->naddmembers + pmix_list_get_size(&plist); PMIX_PROC_CREATE(p, n); if (NULL != coll->sig->addmembers) { memcpy(p, coll->sig->addmembers, coll->sig->naddmembers * sizeof(pmix_proc_t)); } n = coll->sig->naddmembers; PMIX_LIST_FOREACH(nm, &plist, prte_namelist_t) { memcpy(&p[n], &nm->name, sizeof(pmix_proc_t)); ++n; } PMIX_LIST_DESTRUCT(&plist); if (NULL != coll->sig->addmembers) { PMIX_PROC_FREE(coll->sig->addmembers, coll->sig->naddmembers); } coll->sig->addmembers = p; coll->sig->naddmembers = n; coll->nfollowers = n; } } // if they specified a final order, see if one was already given if (NULL != sig->final_order) { if (NULL == coll->sig->final_order) { // cache the directive PMIX_PROC_CREATE(coll->sig->final_order, sig->nfinal); memcpy(coll->sig->final_order, sig->final_order, sig->nfinal * sizeof(pmix_proc_t)); coll->sig->nfinal = sig->nfinal; } else { // see if they match - for now, do a direct match if (coll->sig->nfinal != sig->nfinal) { // this is an error PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return NULL; } if (0 != memcmp(coll->sig->final_order, sig->final_order, sig->nfinal * sizeof(pmix_proc_t))) { // this is an error PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return NULL; } // they are the same, so just ignore the new directive } } if (!coll->sig->assignID && sig->assignID) { coll->sig->assignID = true; } return coll; } } /* if we get here, then this is a new collective - so create * the tracker for it unless directed otherwise */ if (!create) { pmix_output_verbose(1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base: not creating new coll", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); return NULL; } coll = PMIX_NEW(prte_grpcomm_group_t); coll->sig = PMIX_NEW(prte_grpcomm_direct_group_signature_t); coll->sig->op = sig->op; coll->sig->groupID = strdup(sig->groupID); coll->sig->assignID = sig->assignID; // save the participating procs coll->sig->nmembers = sig->nmembers; if (0 < sig->nmembers) { coll->sig->members = (pmix_proc_t *) malloc(coll->sig->nmembers * sizeof(pmix_proc_t)); memcpy(coll->sig->members, sig->members, coll->sig->nmembers * sizeof(pmix_proc_t)); } coll->sig->naddmembers = sig->naddmembers; if (0 < sig->naddmembers) { coll->sig->addmembers = (pmix_proc_t *) malloc(coll->sig->naddmembers * sizeof(pmix_proc_t)); memcpy(coll->sig->addmembers, sig->addmembers, coll->sig->naddmembers * sizeof(pmix_proc_t)); } coll->nfollowers = coll->sig->naddmembers; // need to know the bootstrap in case one is ongoing coll->nleaders = coll->sig->bootstrap; if (0 < sig->bootstrap || sig->follower) { coll->bootstrap = true; } pmix_list_append(&prte_mca_grpcomm_direct_component.group_ops, &coll->super); /* if this is a bootstrap operation, then there is no "rollup" * collective - each daemon reports directly to the DVM controller */ if (coll->bootstrap) { return coll; } /* now get the daemons involved */ if (PRTE_SUCCESS != (rc = create_dmns(sig, &coll->dmns, &coll->ndmns))) { PRTE_ERROR_LOG(rc); pmix_list_remove_item(&prte_mca_grpcomm_direct_component.group_ops, &coll->super); PMIX_RELEASE(coll); return NULL; } /* count the number of contributions we should get */ coll->nexpected = prte_rml_get_num_contributors(coll->dmns, coll->ndmns); /* see if I am in the array of participants - note that I may * be in the rollup tree even though I'm not participating * in the collective itself */ for (n = 0; n < coll->ndmns; n++) { if (coll->dmns[n] == PRTE_PROC_MY_NAME->rank) { coll->nexpected++; break; } } return coll; } static int create_dmns(prte_grpcomm_direct_group_signature_t *sig, pmix_rank_t **dmns, size_t *ndmns) { size_t n; prte_job_t *jdata; prte_proc_t *proc; prte_node_t *node; prte_job_map_t *map; int i; pmix_list_t ds; prte_namelist_t *nm; pmix_rank_t vpid; bool found; size_t nds = 0; pmix_rank_t *dns = NULL; int rc = PRTE_SUCCESS; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group:create_dmns called with %s signature size %" PRIsize_t "", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == sig->members) ? "NULL" : "NON-NULL", sig->nmembers)); PMIX_CONSTRUCT(&ds, pmix_list_t); for (n = 0; n < sig->nmembers; n++) { if (NULL == (jdata = prte_get_job_data_object(sig->members[n].nspace))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; break; } map = (prte_job_map_t*)jdata->map; if (NULL == map || 0 == map->num_nodes) { /* we haven't generated a job map yet - if we are the HNP, * then we should only involve ourselves. Otherwise, we have * no choice but to abort to avoid hangs */ if (PRTE_PROC_IS_MASTER) { rc = PRTE_SUCCESS; break; } PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; break; } if (PMIX_RANK_WILDCARD == sig->members[n].rank) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group::create_dmns called for all procs in job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(sig->members[0].nspace))); /* all daemons hosting this jobid are participating */ for (i = 0; i < map->nodes->size; i++) { if (NULL == (node = pmix_pointer_array_get_item(map->nodes, i))) { continue; } if (NULL == node->daemon) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } found = false; PMIX_LIST_FOREACH(nm, &ds, prte_namelist_t) { if (nm->name.rank == node->daemon->name.rank) { found = true; break; } } if (!found) { PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group::create_dmns adding daemon %s to list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&node->daemon->name))); nm = PMIX_NEW(prte_namelist_t); PMIX_LOAD_PROCID(&nm->name, PRTE_PROC_MY_NAME->nspace, node->daemon->name.rank); pmix_list_append(&ds, &nm->super); } } } else { /* lookup the daemon for this proc and add it to the list */ PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s sign: GETTING PROC OBJECT FOR %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&sig->members[n]))); proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, sig->members[n].rank); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } if (NULL == proc->node || NULL == proc->node->daemon) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } vpid = proc->node->daemon->name.rank; found = false; PMIX_LIST_FOREACH(nm, &ds, prte_namelist_t) { if (nm->name.rank == vpid) { found = true; break; } } if (!found) { nm = PMIX_NEW(prte_namelist_t); PMIX_LOAD_PROCID(&nm->name, PRTE_PROC_MY_NAME->nspace, vpid); pmix_list_append(&ds, &nm->super); } } } done: if (0 < pmix_list_get_size(&ds)) { dns = (pmix_rank_t *) malloc(pmix_list_get_size(&ds) * sizeof(pmix_rank_t)); nds = 0; while (NULL != (nm = (prte_namelist_t *) pmix_list_remove_first(&ds))) { PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:group::create_dmns adding daemon %s to array", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&nm->name))); dns[nds++] = nm->name.rank; PMIX_RELEASE(nm); } } PMIX_LIST_DESTRUCT(&ds); *dmns = dns; *ndmns = nds; return rc; } static int pack_signature(pmix_data_buffer_t *bkt, prte_grpcomm_direct_group_signature_t *sig) { pmix_status_t rc; // pack the operation rc = PMIx_Data_pack(NULL, bkt, &sig->op, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } // add the groupID rc = PMIx_Data_pack(NULL, bkt, &sig->groupID, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } // pack the flag to assign context ID rc = PMIx_Data_pack(NULL, bkt, &sig->assignID, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } // pack the context ID, if one was given rc = PMIx_Data_pack(NULL, bkt, &sig->ctxid_assigned, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (sig->ctxid_assigned) { rc = PMIx_Data_pack(NULL, bkt, &sig->ctxid, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } // pack members, if given rc = PMIx_Data_pack(NULL, bkt, &sig->nmembers, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < sig->nmembers) { rc = PMIx_Data_pack(NULL, bkt, sig->members, sig->nmembers, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } // pack bootstrap number rc = PMIx_Data_pack(NULL, bkt, &sig->bootstrap, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } // pack follower flag rc = PMIx_Data_pack(NULL, bkt, &sig->follower, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } // pack added membership, if given rc = PMIx_Data_pack(NULL, bkt, &sig->naddmembers, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < sig->naddmembers) { rc = PMIx_Data_pack(NULL, bkt, sig->addmembers, sig->naddmembers, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } // pack final order, if given rc = PMIx_Data_pack(NULL, bkt, &sig->nfinal, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < sig->nfinal) { rc = PMIx_Data_pack(NULL, bkt, sig->final_order, sig->nfinal, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } return PRTE_SUCCESS; } static int unpack_signature(pmix_data_buffer_t *buffer, prte_grpcomm_direct_group_signature_t **sig) { pmix_status_t rc; int32_t cnt; prte_grpcomm_direct_group_signature_t *s; s = PMIX_NEW(prte_grpcomm_direct_group_signature_t); // unpack the op cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->op, &cnt, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } // unpack the groupID cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->groupID, &cnt, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } // unpack whether or not to assign a context ID cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->assignID, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } // unpack the context ID, if one was assigned cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->ctxid_assigned, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } if (s->ctxid_assigned) { cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->ctxid, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } } // unpack the membership cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->nmembers, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } if (0 < s->nmembers) { PMIX_PROC_CREATE(s->members, s->nmembers); cnt = s->nmembers; rc = PMIx_Data_unpack(NULL, buffer, s->members, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } } // unpack the bootstrap count cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->bootstrap, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } // unpack the follower flag cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->follower, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } // unpack the added members cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->naddmembers, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } if (0 < s->naddmembers) { PMIX_PROC_CREATE(s->addmembers, s->naddmembers); cnt = s->naddmembers; rc = PMIx_Data_unpack(NULL, buffer, s->addmembers, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } } // unpack the final order cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->nfinal, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } if (0 < s->nfinal) { PMIX_PROC_CREATE(s->final_order, s->nfinal); cnt = s->nfinal; rc = PMIx_Data_unpack(NULL, buffer, s->final_order, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } } *sig = s; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct_component.c0000664000175000017500000001143615203542132025335 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/mca.h" #include "src/runtime/prte_globals.h" #include "src/util/proc_info.h" #include "grpcomm_direct.h" static int direct_query(pmix_mca_base_module_t **module, int *priority); /* * Struct of function pointers that need to be initialized */ prte_grpcomm_direct_component_t prte_mca_grpcomm_direct_component = { .super = { PRTE_GRPCOMM_BASE_VERSION_4_0_0, .pmix_mca_component_name = "direct", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), .pmix_mca_query_component = direct_query, }, .fence_ops = PMIX_LIST_STATIC_INIT, .group_ops = PMIX_LIST_STATIC_INIT }; PMIX_MCA_BASE_COMPONENT_INIT(prte, grpcomm, direct) static int direct_query(pmix_mca_base_module_t **module, int *priority) { /* we are always available */ *priority = 5; *module = (pmix_mca_base_module_t *) &prte_grpcomm_direct_module; return PRTE_SUCCESS; } static void scon(prte_grpcomm_direct_fence_signature_t *p) { p->signature = NULL; p->sz = 0; } static void sdes(prte_grpcomm_direct_fence_signature_t *p) { if (NULL != p->signature) { free(p->signature); } } PMIX_CLASS_INSTANCE(prte_grpcomm_direct_fence_signature_t, pmix_object_t, scon, sdes); static void sgcon(prte_grpcomm_direct_group_signature_t *p)\ { p->op = PMIX_GROUP_NONE; p->groupID = NULL; p->assignID = false; p->ctxid = 0; p->ctxid_assigned = false; p->members = NULL; p->nmembers = 0; p->bootstrap = 0; p->follower = false; p->addmembers = NULL; p->naddmembers = 0; p->final_order = NULL; p->nfinal = 0; } static void sgdes(prte_grpcomm_direct_group_signature_t *p) { if (NULL != p->groupID) { free(p->groupID); } if (NULL != p->members) { free(p->members); } if (NULL != p->addmembers) { free(p->addmembers); } } PMIX_CLASS_INSTANCE(prte_grpcomm_direct_group_signature_t, pmix_object_t, sgcon, sgdes); static void ccon(prte_grpcomm_fence_t *p) { p->sig = NULL; p->status = PMIX_SUCCESS; PMIX_DATA_BUFFER_CONSTRUCT(&p->bucket); p->dmns = NULL; p->ndmns = 0; p->nexpected = 0; p->nreported = 0; p->timeout = 0; p->cbfunc = NULL; p->cbdata = NULL; } static void cdes(prte_grpcomm_fence_t *p) { if (NULL != p->sig) { PMIX_RELEASE(p->sig); } PMIX_DATA_BUFFER_DESTRUCT(&p->bucket); if (NULL != p->dmns) { free(p->dmns); } } PMIX_CLASS_INSTANCE(prte_grpcomm_fence_t, pmix_list_item_t, ccon, cdes); static void gccon(prte_grpcomm_group_t *p) { p->sig = NULL; p->status = PMIX_SUCCESS; p->dmns = NULL; p->ndmns = 0; p->bootstrap = false; p->nexpected = 0; p->nreported = 0; p->nleaders = 0; p->nleaders_reported = 0; p->nfollowers = 0; p->nfollowers_reported = 0; p->assignID = false; p->timeout = 0; p->memsize = 0; p->grpinfo = PMIx_Info_list_start(); p->endpts = PMIx_Info_list_start(); p->cbfunc = NULL; p->cbdata = NULL; } static void gcdes(prte_grpcomm_group_t *p) { if (NULL != p->sig) { PMIX_RELEASE(p->sig); } PMIx_Info_list_release(p->grpinfo); PMIx_Info_list_release(p->endpts); if (NULL != p->dmns) { free(p->dmns); } } PMIX_CLASS_INSTANCE(prte_grpcomm_group_t, pmix_list_item_t, gccon, gcdes); static void mdcon(prte_pmix_fence_caddy_t *p) { p->sig = NULL; p->buf = NULL; p->procs = NULL; p->nprocs = 0; p->info = NULL; p->ninfo = 0; p->data = NULL; p->ndata = 0; p->cbfunc = NULL; p->cbdata = NULL; } static void mddes(prte_pmix_fence_caddy_t *p) { if (NULL != p->sig) { PMIX_RELEASE(p->sig); } if (NULL != p->buf) { PMIX_DATA_BUFFER_RELEASE(p->buf); } } PMIX_CLASS_INSTANCE(prte_pmix_fence_caddy_t, pmix_object_t, mdcon, mddes); prrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct_xcast.c0000664000175000017500000003042715203542132024456 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; -*- */ /* * Copyright (c) 2007 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All * rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/util/name_fns.h" #include "src/util/nidmap.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "grpcomm_direct.h" #include "src/mca/grpcomm/base/base.h" static int pack_xcast(pmix_data_buffer_t *buffer, pmix_data_buffer_t *message, prte_rml_tag_t tag); int prte_grpcomm_direct_xcast(prte_rml_tag_t tag, pmix_data_buffer_t *msg) { int rc; pmix_data_buffer_t *buf; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:xcast: with %d bytes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) msg->bytes_used)); /* this function does not access any framework-global data, and * so it does not require us to push it into the event library */ /* prep the output buffer */ PMIX_DATA_BUFFER_CREATE(buf); /* setup the payload */ if (PRTE_SUCCESS != (rc = pack_xcast(buf, msg, tag))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } /* send it to the HNP (could be myself) for relay */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, buf, PRTE_RML_TAG_XCAST); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return rc; } return PRTE_SUCCESS; } void prte_grpcomm_direct_xcast_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tg, void *cbdata) { prte_routed_tree_t *nm; int ret, cnt; pmix_data_buffer_t *relay = NULL, *rly, *rlycopy; pmix_data_buffer_t datbuf, *data; bool compressed; prte_job_t *daemons; pmix_list_t coll; prte_rml_tag_t tag; pmix_byte_object_t bo, pbo; pmix_value_t val; pmix_proc_t dmn; PRTE_HIDE_UNUSED_PARAMS(status, sender, tg, cbdata); PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:xcast:recv: with %d bytes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) buffer->bytes_used)); /* we need a passthru buffer to send to our children - we leave it * as compressed data */ PMIX_DATA_BUFFER_CREATE(rly); ret = PMIx_Data_copy_payload(rly, buffer); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_RELEASE(rly); return; } PMIX_DATA_BUFFER_CONSTRUCT(&datbuf); /* setup the relay list */ PMIX_CONSTRUCT(&coll, pmix_list_t); /* unpack the flag to see if this payload is compressed */ cnt = 1; ret = PMIx_Data_unpack(NULL, buffer, &compressed, &cnt, PMIX_BOOL); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } /* unpack the data blob */ cnt = 1; ret = PMIx_Data_unpack(NULL, buffer, &pbo, &cnt, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } if (compressed) { /* decompress the data */ if (PMIx_Data_decompress((uint8_t *) pbo.bytes, pbo.size, (uint8_t **) &bo.bytes, &bo.size)) { /* the data has been uncompressed */ ret = PMIx_Data_load(&datbuf, &bo); if (PMIX_SUCCESS != ret) { PMIX_BYTE_OBJECT_DESTRUCT(&pbo); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } } else { pmix_show_help("help-prte-runtime.txt", "failed-to-uncompress", true, prte_process_info.nodename); PMIX_BYTE_OBJECT_DESTRUCT(&pbo); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } } else { ret = PMIx_Data_load(&datbuf, &pbo); if (PMIX_SUCCESS != ret) { PMIX_BYTE_OBJECT_DESTRUCT(&pbo); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } } PMIX_BYTE_OBJECT_DESTRUCT(&pbo); data = &datbuf; /* get the target tag */ cnt = 1; ret = PMIx_Data_unpack(NULL, data, &tag, &cnt, PMIX_UINT32); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); return; } /* copy the msg for relay to ourselves */ PMIX_DATA_BUFFER_CREATE(relay); ret = PMIx_Data_copy_payload(relay, data); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); PMIX_DATA_BUFFER_RELEASE(relay); return; } if (PRTE_RML_TAG_WIREUP == tag && !PRTE_PROC_IS_MASTER) { if (PRTE_SUCCESS != (ret = prte_util_decode_nidmap(data))) { PRTE_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); PMIX_DATA_BUFFER_RELEASE(relay); return; } /* unpack the wireup info */ cnt = 1; while (PMIX_SUCCESS == (ret = PMIx_Data_unpack(NULL, data, &dmn, &cnt, PMIX_PROC))) { PMIX_VALUE_CONSTRUCT(&val); val.type = PMIX_STRING; cnt = 1; ret = PMIx_Data_unpack(NULL, data, &val.data.string, &cnt, PMIX_STRING); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); PMIX_DATA_BUFFER_RELEASE(relay); return; } if (!PMIX_CHECK_PROCID(&dmn, PRTE_PROC_MY_HNP) && !PMIX_CHECK_PROCID(&dmn, PRTE_PROC_MY_NAME) && !PMIX_CHECK_PROCID(&dmn, PRTE_PROC_MY_PARENT)) { /* store it locally */ ret = PMIx_Store_internal(&dmn, PMIX_PROC_URI, &val); PMIX_VALUE_DESTRUCT(&val); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); PMIX_DATA_BUFFER_DESTRUCT(&datbuf); PMIX_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); PMIX_DATA_BUFFER_RELEASE(relay); return; } } } if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != ret) { PMIX_ERROR_LOG(ret); } } daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); if (!prte_get_attribute(&daemons->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* send the message to each of our children */ PMIX_LIST_FOREACH(nm, &prte_rml_base.children, prte_routed_tree_t) { PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:send_relay sending relay msg of %d bytes to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) rly->bytes_used, PRTE_VPID_PRINT(nm->rank))); /* copy the buffer for send */ PMIX_DATA_BUFFER_CREATE(rlycopy); ret = PMIx_Data_copy_payload(rlycopy, rly); if (PMIX_SUCCESS != ret) { PRTE_ERROR_LOG(ret); PMIX_DATA_BUFFER_RELEASE(rlycopy); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); continue; } PRTE_RML_SEND(ret, nm->rank, rlycopy, PRTE_RML_TAG_XCAST); if (PRTE_SUCCESS != ret) { PRTE_ERROR_LOG(ret); PMIX_DATA_BUFFER_RELEASE(rlycopy); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); continue; } } } /* cleanup */ PMIX_LIST_DESTRUCT(&coll); PMIX_DATA_BUFFER_RELEASE(rly); // retain accounting /* now pass the relay buffer to myself for processing IFF it * wasn't just a wireup message - don't * inject it into the RML system via send as that will compete * with the relay messages down in the OOB. Instead, pass it * directly to the RML message processor */ if (PRTE_RML_TAG_WIREUP != tag) { PRTE_RML_POST_MESSAGE(PRTE_PROC_MY_NAME, tag, 1, relay->base_ptr, relay->bytes_used); relay->base_ptr = NULL; relay->bytes_used = 0; } if (NULL != relay) { PMIX_DATA_BUFFER_RELEASE(relay); } PMIX_DATA_BUFFER_DESTRUCT(&datbuf); } static int pack_xcast(pmix_data_buffer_t *buffer, pmix_data_buffer_t *message, prte_rml_tag_t tag) { int rc; pmix_data_buffer_t data; bool compressed; pmix_byte_object_t bo; size_t sz; /* setup an intermediate buffer */ PMIX_DATA_BUFFER_CONSTRUCT(&data); /* pass the final tag */ rc = PMIx_Data_pack(NULL, &data, &tag, 1, PRTE_RML_TAG); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&data); return rc; } /* copy the payload into the new buffer - this is non-destructive, so our * caller is still responsible for releasing any memory in the buffer they * gave to us */ rc = PMIx_Data_copy_payload(&data, message); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&data); return rc; } /* see if we want to compress this message */ if (PMIx_Data_compress((uint8_t *) data.base_ptr, data.bytes_used, (uint8_t **) &bo.bytes, &sz)) { /* the data was compressed - mark that we compressed it */ compressed = true; bo.size = sz; } else { /* mark that it was not compressed */ compressed = false; bo.bytes = data.base_ptr; bo.size = data.bytes_used; data.base_ptr = NULL; data.bytes_used = 0; } PMIX_DATA_BUFFER_DESTRUCT(&data); rc = PMIx_Data_pack(NULL, buffer, &compressed, 1, PMIX_BOOL); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_BYTE_OBJECT_DESTRUCT(&bo); return rc; } rc = PMIx_Data_pack(NULL, buffer, &bo, 1, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_BYTE_OBJECT_DESTRUCT(&bo); return rc; } PMIX_BYTE_OBJECT_DESTRUCT(&bo); return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/grpcomm/direct/Makefile.am0000664000175000017500000000255015203542132022142 0ustar alastairalastair# # Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2013 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2022-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AM_CPPFLAGS = $(grpcomm_direct_CPPFLAGS) sources = \ grpcomm_direct.h \ grpcomm_direct.c \ grpcomm_direct_xcast.c \ grpcomm_direct_fence.c \ grpcomm_direct_group.c \ grpcomm_direct_component.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_grpcomm_direct_DSO component_noinst = component_install = prte_mca_grpcomm_direct.la else component_noinst = libprtemca_grpcomm_direct.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_grpcomm_direct_la_SOURCES = $(sources) prte_mca_grpcomm_direct_la_LDFLAGS = -module -avoid-version prte_mca_grpcomm_direct_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_grpcomm_direct_la_SOURCES =$(sources) libprtemca_grpcomm_direct_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct.c0000664000175000017500000000605615203542132023255 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; -*- */ /* * Copyright (c) 2007 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All * rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/mca/state/state.h" #include "src/util/name_fns.h" #include "src/util/nidmap.h" #include "src/util/proc_info.h" #include "src/util/pmix_show_help.h" #include "grpcomm_direct.h" #include "src/mca/grpcomm/base/base.h" /* Static API's */ static int init(void); static void finalize(void); /* Module def */ prte_grpcomm_base_module_t prte_grpcomm_direct_module = { .init = init, .finalize = finalize, .xcast = prte_grpcomm_direct_xcast, .fence = prte_grpcomm_direct_fence, .group = prte_grpcomm_direct_group }; /** * Initialize the module */ static int init(void) { /* setup the trackers */ PMIX_CONSTRUCT(&prte_mca_grpcomm_direct_component.fence_ops, pmix_list_t); PMIX_CONSTRUCT(&prte_mca_grpcomm_direct_component.group_ops, pmix_list_t); /* xcast receive */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_XCAST, PRTE_RML_PERSISTENT, prte_grpcomm_direct_xcast_recv, NULL); /* fence receives */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_FENCE, PRTE_RML_PERSISTENT, prte_grpcomm_direct_fence_recv, NULL); /* setup recv for barrier release */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_FENCE_RELEASE, PRTE_RML_PERSISTENT, prte_grpcomm_direct_fence_release, NULL); /* group receives */ PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_GROUP, PRTE_RML_PERSISTENT, prte_grpcomm_direct_grp_recv, NULL); PRTE_RML_RECV(PRTE_NAME_WILDCARD, PRTE_RML_TAG_GROUP_RELEASE, PRTE_RML_PERSISTENT, prte_grpcomm_direct_grp_release, NULL); return PRTE_SUCCESS; } /** * Finalize the module */ static void finalize(void) { PMIX_LIST_DESTRUCT(&prte_mca_grpcomm_direct_component.fence_ops); PMIX_LIST_DESTRUCT(&prte_mca_grpcomm_direct_component.group_ops); PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_XCAST); PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_FENCE); PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_FENCE_RELEASE); PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_GROUP); PRTE_RML_CANCEL(PRTE_NAME_WILDCARD, PRTE_RML_TAG_GROUP_RELEASE); return; } prrte-4.1.1~rc2/src/mca/grpcomm/direct/grpcomm_direct_fence.c0000664000175000017500000005640415203542132024417 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; -*- */ /* * Copyright (c) 2007 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2011-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All * rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/rml/rml.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/mca/state/state.h" #include "src/util/name_fns.h" #include "src/util/nidmap.h" #include "src/util/proc_info.h" #include "grpcomm_direct.h" #include "src/mca/grpcomm/base/base.h" /* internal functions */ static void fence(int sd, short args, void *cbdata); static prte_grpcomm_fence_t* get_tracker(prte_grpcomm_direct_fence_signature_t *sig, bool create); static int create_dmns(prte_grpcomm_direct_fence_signature_t *sig, pmix_rank_t **dmns, size_t *ndmns); static int fence_sig_pack(pmix_data_buffer_t *bkt, prte_grpcomm_direct_fence_signature_t *sig); static int fence_sig_unpack(pmix_data_buffer_t *buffer, prte_grpcomm_direct_fence_signature_t **sig); int prte_grpcomm_direct_fence(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { prte_pmix_fence_caddy_t *cd; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); // bozo check if (NULL == procs) { return PRTE_ERR_NOT_SUPPORTED; } cd = PMIX_NEW(prte_pmix_fence_caddy_t); cd->procs = (pmix_proc_t*)procs; cd->nprocs = nprocs; cd->info = (pmix_info_t*)info; cd->ninfo = ninfo; cd->data = data; cd->ndata = ndata; cd->cbfunc = cbfunc; cd->cbdata = cbdata; /* must push this into the event library to ensure we can * access framework-global data safely */ prte_event_set(prte_event_base, &cd->ev, -1, PRTE_EV_WRITE, fence, cd); PMIX_POST_OBJECT(cd); prte_event_active(&cd->ev, PRTE_EV_WRITE, 1); return PRTE_SUCCESS; } static void fence(int sd, short args, void *cbdata) { prte_pmix_fence_caddy_t *cd = (prte_pmix_fence_caddy_t *) cbdata; prte_grpcomm_direct_fence_signature_t sig; prte_grpcomm_fence_t *coll; int rc; pmix_data_buffer_t *relay, bkt; pmix_byte_object_t bo; PRTE_HIDE_UNUSED_PARAMS(sd, args); PMIX_ACQUIRE_OBJECT(cd); PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct: fence", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* compute the signature of this collective */ PMIX_CONSTRUCT(&sig, prte_grpcomm_direct_fence_signature_t); sig.sz = cd->nprocs; sig.signature = (pmix_proc_t *) malloc(sig.sz * sizeof(pmix_proc_t)); memcpy(sig.signature, cd->procs, sig.sz * sizeof(pmix_proc_t)); /* retrieve an existing tracker, create it if not * already found. The fence module is responsible * for releasing it upon completion of the collective */ coll = get_tracker(&sig, true); if (NULL == coll) { PMIX_DESTRUCT(&sig); PMIX_RELEASE(cd); return; } coll->cbfunc = cd->cbfunc; coll->cbdata = cd->cbdata; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct: fence", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); // execute the fence operation PMIX_DATA_BUFFER_CREATE(relay); /* pack the signature */ rc = fence_sig_pack(relay, coll->sig); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(relay); PMIX_RELEASE(cd); return; } // pack the info structs rc = PMIx_Data_pack(NULL, relay, &cd->ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(relay); PMIX_RELEASE(cd); return; } if (0 < cd->ninfo) { rc = PMIx_Data_pack(NULL, relay, cd->info, cd->ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(relay); PMIX_RELEASE(cd); return; } } /* pass along the payload */ PMIX_DATA_BUFFER_CONSTRUCT(&bkt); bo.bytes = cd->data; bo.size = cd->ndata; PMIx_Data_embed(&bkt, &bo); rc = PMIx_Data_copy_payload(relay, &bkt); PMIX_DATA_BUFFER_DESTRUCT(&bkt); if (PMIX_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(relay); PMIX_RELEASE(cd); return; } /* send this to ourselves for processing */ PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence sending to ourself", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_RML_SEND(rc, PRTE_PROC_MY_NAME->rank, relay, PRTE_RML_TAG_FENCE); PMIX_RELEASE(cd); return; } void prte_grpcomm_direct_fence_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { int32_t cnt; int rc, timeout; size_t n, ninfo; pmix_status_t st; pmix_info_t *info = NULL; prte_grpcomm_direct_fence_signature_t *sig = NULL; pmix_data_buffer_t *reply; prte_grpcomm_fence_t *coll; PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata); PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct fence recvd from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender))); /* unpack the signature */ rc = fence_sig_unpack(buffer, &sig); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* check for the tracker and create it if not found */ if (NULL == (coll = get_tracker(sig, true))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PMIX_RELEASE(sig); return; } PMIX_RELEASE(sig); // unpack the info structs cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &ninfo, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } if (0 < ninfo) { PMIX_INFO_CREATE(info, ninfo); cnt = ninfo; rc = PMIx_Data_unpack(NULL, buffer, info, &cnt, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); return; } } /* cycle thru the info to look for keys we support */ for (n=0; n < ninfo; n++) { if (PMIX_CHECK_KEY(&info[n], PMIX_TIMEOUT)) { PMIX_VALUE_GET_NUMBER(rc, &info[n].value, timeout, int); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); return; } if (coll->timeout < timeout) { coll->timeout = timeout; } /* update the info with the collected value */ info[n].value.type = PMIX_INT; info[n].value.data.integer = coll->timeout; } else if (PMIX_CHECK_KEY(&info[n], PMIX_LOCAL_COLLECTIVE_STATUS)) { PMIX_VALUE_GET_NUMBER(rc, &info[n].value, st, pmix_status_t); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); return; } if (PMIX_SUCCESS != st && PMIX_SUCCESS == coll->status) { coll->status = st; } /* update the info with the collected value */ info[n].value.type = PMIX_STATUS; info[n].value.data.status = coll->status; } } /* increment nprocs reported for collective */ coll->nreported++; // transfer any data rc = PMIx_Data_copy_payload(&coll->bucket, buffer); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_INFO_FREE(info, ninfo); return; } PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct fence recv nexpected %d nrep %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) coll->nexpected, (int) coll->nreported)); /* see if everyone has reported */ if (coll->nreported == coll->nexpected) { if (PRTE_PROC_IS_MASTER) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct fence HNP reports complete", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* the allgather is complete - send the xcast */ PMIX_DATA_BUFFER_CREATE(reply); /* pack the signature */ rc = fence_sig_pack(reply, coll->sig); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } /* pack the status */ rc = PMIx_Data_pack(NULL, reply, &coll->status, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } /* transfer the collected bucket */ rc = PMIx_Data_copy_payload(reply, &coll->bucket); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } /* send the release via xcast */ (void) prte_grpcomm.xcast(PRTE_RML_TAG_FENCE_RELEASE, reply); } else { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct fence rollup complete - sending to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(PRTE_PROC_MY_PARENT))); PMIX_DATA_BUFFER_CREATE(reply); /* pack the signature */ rc = fence_sig_pack(reply, coll->sig); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } // pack the info structs rc = PMIx_Data_pack(NULL, reply, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } if (0 < ninfo) { rc = PMIx_Data_pack(NULL, reply, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(reply); PMIX_INFO_FREE(info, ninfo); return; } } PMIX_INFO_FREE(info, ninfo); /* transfer the collected bucket */ rc = PMIx_Data_copy_payload(reply, &coll->bucket); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return; } /* send the info to our parent */ PRTE_RML_SEND(rc, PRTE_PROC_MY_PARENT->rank, reply, PRTE_RML_TAG_FENCE); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return; } } } } static void relcb(void *cbdata) { uint8_t *data = (uint8_t *) cbdata; if (NULL != data) { free(data); } } void prte_grpcomm_direct_fence_release(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { int32_t cnt; int rc, ret; prte_grpcomm_direct_fence_signature_t *sig = NULL; prte_grpcomm_fence_t *coll; pmix_byte_object_t bo; PRTE_HIDE_UNUSED_PARAMS(status, sender, tag, cbdata); PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct: fence release called with %d bytes", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (int) buffer->bytes_used)); /* unpack the signature */ rc = fence_sig_unpack(buffer, &sig); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return; } /* unpack the return status */ cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &ret, &cnt, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(sig); return; } /* check for the tracker - it is not an error if not * found as that just means we are not involved * in the collective */ if (NULL == (coll = get_tracker(sig, false))) { PMIX_RELEASE(sig); return; } /* unload the buffer */ PMIX_BYTE_OBJECT_CONSTRUCT(&bo); rc = PMIx_Data_unload(buffer, &bo); if (PMIX_SUCCESS != rc) { ret = rc; } /* execute the callback */ if (NULL != coll->cbfunc) { coll->cbfunc(ret, bo.bytes, bo.size, coll->cbdata, relcb, bo.bytes); } pmix_list_remove_item(&prte_mca_grpcomm_direct_component.fence_ops, &coll->super); PMIX_RELEASE(coll); PMIX_RELEASE(sig); } static prte_grpcomm_fence_t* get_tracker(prte_grpcomm_direct_fence_signature_t *sig, bool create) { prte_grpcomm_fence_t *coll; int rc; size_t n; /* search the existing tracker list to see if this already exists */ PMIX_LIST_FOREACH(coll, &prte_mca_grpcomm_direct_component.fence_ops, prte_grpcomm_fence_t) { if (sig->sz == coll->sig->sz) { // must match proc signature if (0 == memcmp(sig->signature, coll->sig->signature, sig->sz * sizeof(pmix_proc_t))) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base:returning existing collective", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); return coll; } } } /* if we get here, then this is a new collective - so create * the tracker for it */ if (!create) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:base: not creating new coll", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); return NULL; } coll = PMIX_NEW(prte_grpcomm_fence_t); // we have to know the participating procs coll->sig = PMIX_NEW(prte_grpcomm_direct_fence_signature_t); coll->sig->sz = sig->sz; coll->sig->signature = (pmix_proc_t *) malloc(coll->sig->sz * sizeof(pmix_proc_t)); memcpy(coll->sig->signature, sig->signature, coll->sig->sz * sizeof(pmix_proc_t)); pmix_list_append(&prte_mca_grpcomm_direct_component.fence_ops, &coll->super); /* now get the daemons involved */ if (PRTE_SUCCESS != (rc = create_dmns(sig, &coll->dmns, &coll->ndmns))) { PRTE_ERROR_LOG(rc); return NULL; } /* count the number of contributions we should get */ coll->nexpected = prte_rml_get_num_contributors(coll->dmns, coll->ndmns); /* see if I am in the array of participants - note that I may * be in the rollup tree even though I'm not participating * in the collective itself */ for (n = 0; n < coll->ndmns; n++) { if (coll->dmns[n] == PRTE_PROC_MY_NAME->rank) { coll->nexpected++; break; } } return coll; } static int create_dmns(prte_grpcomm_direct_fence_signature_t *sig, pmix_rank_t **dmns, size_t *ndmns) { size_t n; prte_job_t *jdata; prte_proc_t *proc; prte_node_t *node; prte_job_map_t *map; int i; pmix_list_t ds; prte_namelist_t *nm; pmix_rank_t vpid; bool found; size_t nds = 0; pmix_rank_t *dns = NULL; int rc = PRTE_SUCCESS; PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence:create_dmns called with %s signature size %" PRIsize_t "", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == sig->signature) ? "NULL" : "NON-NULL", sig->sz)); /* if the target jobid is our own, * then all daemons are participating */ if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, sig->signature[0].nspace)) { *ndmns = prte_process_info.num_daemons; *dmns = NULL; return PRTE_SUCCESS; } PMIX_CONSTRUCT(&ds, pmix_list_t); for (n = 0; n < sig->sz; n++) { if (NULL == (jdata = prte_get_job_data_object(sig->signature[n].nspace))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; break; } map = (prte_job_map_t*)jdata->map; if (NULL == map || 0 == map->num_nodes) { /* we haven't generated a job map yet - if we are the HNP, * then we should only involve ourselves. Otherwise, we have * no choice but to abort to avoid hangs */ if (PRTE_PROC_IS_MASTER) { rc = PRTE_SUCCESS; break; } PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; break; } if (PMIX_RANK_WILDCARD == sig->signature[n].rank) { PMIX_OUTPUT_VERBOSE((1, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence::create_dmns called for all procs in job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(sig->signature[0].nspace))); /* all daemons hosting this jobid are participating */ for (i = 0; i < map->nodes->size; i++) { if (NULL == (node = pmix_pointer_array_get_item(map->nodes, i))) { continue; } if (NULL == node->daemon) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } found = false; PMIX_LIST_FOREACH(nm, &ds, prte_namelist_t) { if (nm->name.rank == node->daemon->name.rank) { found = true; break; } } if (!found) { PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence::create_dmns adding daemon %s to list", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&node->daemon->name))); nm = PMIX_NEW(prte_namelist_t); PMIX_LOAD_PROCID(&nm->name, PRTE_PROC_MY_NAME->nspace, node->daemon->name.rank); pmix_list_append(&ds, &nm->super); } } } else { /* lookup the daemon for this proc and add it to the list */ PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s sign: GETTING PROC OBJECT FOR %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&sig->signature[n]))); proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, sig->signature[n].rank); if (NULL == proc) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } if (NULL == proc->node || NULL == proc->node->daemon) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto done; } vpid = proc->node->daemon->name.rank; found = false; PMIX_LIST_FOREACH(nm, &ds, prte_namelist_t) { if (nm->name.rank == vpid) { found = true; break; } } if (!found) { nm = PMIX_NEW(prte_namelist_t); PMIX_LOAD_PROCID(&nm->name, PRTE_PROC_MY_NAME->nspace, vpid); pmix_list_append(&ds, &nm->super); } } } done: if (0 < pmix_list_get_size(&ds)) { dns = (pmix_rank_t *) malloc(pmix_list_get_size(&ds) * sizeof(pmix_rank_t)); nds = 0; while (NULL != (nm = (prte_namelist_t *) pmix_list_remove_first(&ds))) { PMIX_OUTPUT_VERBOSE((5, prte_grpcomm_base_framework.framework_output, "%s grpcomm:direct:fence::create_dmns adding daemon %s to array", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&nm->name))); dns[nds++] = nm->name.rank; PMIX_RELEASE(nm); } } PMIX_LIST_DESTRUCT(&ds); *dmns = dns; *ndmns = nds; return rc; } static int fence_sig_pack(pmix_data_buffer_t *bkt, prte_grpcomm_direct_fence_signature_t *sig) { pmix_status_t rc; // always send the participating procs rc = PMIx_Data_pack(NULL, bkt, &sig->sz, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } if (0 < sig->sz) { rc = PMIx_Data_pack(NULL, bkt, sig->signature, sig->sz, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return prte_pmix_convert_status(rc); } } return PRTE_SUCCESS; } static int fence_sig_unpack(pmix_data_buffer_t *buffer, prte_grpcomm_direct_fence_signature_t **sig) { pmix_status_t rc; int32_t cnt; prte_grpcomm_direct_fence_signature_t *s; s = PMIX_NEW(prte_grpcomm_direct_fence_signature_t); // unpack the participating procs cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &s->sz, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } if (0 < s->sz) { PMIX_PROC_CREATE(s->signature, s->sz); cnt = s->sz; rc = PMIx_Data_unpack(NULL, buffer, s->signature, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(s); return prte_pmix_convert_status(rc); } } *sig = s; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/grpcomm/grpcomm.h0000664000175000017500000001130615203542132020450 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The PRTE Group Communications * * The PRTE Group Comm framework provides communication services that * span entire jobs or collections of processes. It is not intended to be * used for point-to-point communications (the RML does that), nor should * it be viewed as a high-performance communication channel for large-scale * data transfers. */ #ifndef MCA_GRPCOMM_H #define MCA_GRPCOMM_H /* * includes */ #include "prte_config.h" #include "constants.h" #include "types.h" #include "src/class/pmix_bitmap.h" #include "src/class/pmix_list.h" #include "src/pmix/pmix-internal.h" #include "src/mca/mca.h" #include "src/rml/rml_types.h" BEGIN_C_DECLS typedef struct { pmix_object_t super; prte_event_t ev; pmix_lock_t lock; pmix_group_operation_t op; char *grpid; const pmix_proc_t *procs; size_t nprocs; const pmix_info_t *directives; size_t ndirs; pmix_info_t *info; size_t ninfo; pmix_info_cbfunc_t cbfunc; void *cbdata; } prte_pmix_grp_caddy_t; PMIX_CLASS_DECLARATION(prte_pmix_grp_caddy_t); /* define a callback function to be invoked upon * collective completion */ typedef void (*prte_grpcomm_cbfunc_t)(int status, pmix_data_buffer_t *buf, void *cbdata); /* * Component functions - all MUST be provided! */ /* initialize the selected module */ typedef int (*prte_grpcomm_base_module_init_fn_t)(void); /* finalize the selected module */ typedef void (*prte_grpcomm_base_module_finalize_fn_t)(void); /* Scalably send a message. */ typedef int (*prte_grpcomm_base_module_xcast_fn_t)(prte_rml_tag_t tag, pmix_data_buffer_t *msg); /* fence - gather data from all specified procs. Barrier operations * will provide NULL data. * * NOTE: this is a non-blocking call. The callback function * will be invoked upon completion. */ typedef int (*prte_grpcomm_base_module_fence_fn_t)(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata); /* support group operations - this is basically a fence * operation, but there are enough differences to warrant keeping it * separate to avoid over-complicating the fence code */ typedef int (*prte_grpcomm_base_module_grp_fn_t)(pmix_group_operation_t op, char *grpid, const pmix_proc_t procs[], size_t nprocs, const pmix_info_t directives[], size_t ndirs, pmix_info_cbfunc_t cbfunc, void *cbdata); /* * Ver 4.0 */ typedef struct { prte_grpcomm_base_module_init_fn_t init; prte_grpcomm_base_module_finalize_fn_t finalize; /* collective operations */ prte_grpcomm_base_module_xcast_fn_t xcast; prte_grpcomm_base_module_fence_fn_t fence; prte_grpcomm_base_module_grp_fn_t group; } prte_grpcomm_base_module_t; /* * the standard component data structure */ typedef pmix_mca_base_component_t prte_grpcomm_base_component_t; /* * Macro for use in components that are of type grpcomm v3.0.0 */ #define PRTE_GRPCOMM_BASE_VERSION_4_0_0 \ /* grpcomm v4.0 is chained to MCA v2.0 */ \ PRTE_MCA_BASE_VERSION_3_0_0("grpcomm", 4, 0, 0) /* Global structure for accessing grpcomm functions */ PRTE_EXPORT extern prte_grpcomm_base_module_t prte_grpcomm; END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/state/0000775000175000017500000000000015203542132016306 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/state/state_types.h0000664000175000017500000000476415203542132021036 0ustar alastairalastair/* * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /**** PRTE STATE MACHINE ****/ /* States are treated as events so that the event * library can sequence them. Each state consists * of an event, a job or process state, a pointer * to the respective object, and a callback function * to be executed for that state. Events can be defined * at different priorities - e.g., SYS priority for * events associated with launching jobs, and ERR priority * for events associated with abnormal termination of * a process. * * The state machine consists of a list of state objects, * each defining a state-cbfunc pair. At startup, a default * list is created by the base functions which is then * potentially customized by selected components within * the various PRTE frameworks. For example, a PLM component * may need to insert states in the launch procedure, or may * want to redirect a particular state callback to a custom * function. * * For convenience, an ANY state can be defined along with a generic * callback function, with the corresponding state object * placed at the end of the state machine. Setting the * machine to a state that has not been explicitly defined * will cause this default action to be executed. Thus, you * don't have to explicitly define a state-cbfunc pair * for every job or process state. */ #ifndef _PRTE_STATE_TYPES_H_ #define _PRTE_STATE_TYPES_H_ #include "prte_config.h" #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" BEGIN_C_DECLS typedef void (*prte_state_cbfunc_t)(int fd, short args, void *cb); typedef struct { pmix_list_item_t super; prte_job_state_t job_state; prte_proc_state_t proc_state; prte_state_cbfunc_t cbfunc; } prte_state_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_state_t); /* caddy for passing job and proc data to state event handlers */ typedef struct { pmix_object_t super; prte_event_t ev; prte_job_t *jdata; prte_job_state_t job_state; pmix_proc_t name; prte_proc_state_t proc_state; } prte_state_caddy_t; PRTE_EXPORT PMIX_CLASS_DECLARATION(prte_state_caddy_t); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/state/dvm/0000775000175000017500000000000015203542132017074 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/state/dvm/owner.txt0000664000175000017500000000023715203542132020771 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: active prrte-4.1.1~rc2/src/mca/state/dvm/state_dvm_component.c0000664000175000017500000000440515203542132023313 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/util/pmix_output.h" #include "src/mca/state/base/base.h" #include "src/mca/state/state.h" #include "state_dvm.h" /* * Public string for version number */ const char *prte_mca_state_dvm_component_version_string = "PRTE STATE dvm MCA component version " PRTE_VERSION; /* * Local functionality */ static int state_dvm_open(void); static int state_dvm_close(void); static int state_dvm_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointer to our public functions in it */ prte_state_base_component_t prte_mca_state_dvm_component = { PRTE_STATE_BASE_VERSION_1_0_0, /* Component name and version */ .pmix_mca_component_name = "dvm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = state_dvm_open, .pmix_mca_close_component = state_dvm_close, .pmix_mca_query_component = state_dvm_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, state, dvm) static int state_dvm_open(void) { return PRTE_SUCCESS; } static int state_dvm_close(void) { return PRTE_SUCCESS; } static int state_dvm_component_query(pmix_mca_base_module_t **module, int *priority) { /* used by DVM masters */ if (PRTE_PROC_IS_MASTER) { *priority = 100; *module = (pmix_mca_base_module_t *) &prte_state_dvm_module; return PRTE_SUCCESS; } *priority = 0; *module = NULL; return PRTE_ERR_NOT_AVAILABLE; } prrte-4.1.1~rc2/src/mca/state/dvm/state_dvm.h0000664000175000017500000000146515203542132021241 0ustar alastairalastair/* * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef MCA_STATE_DVM_EXPORT_H #define MCA_STATE_DVM_EXPORT_H #include "prte_config.h" #include "src/mca/state/state.h" BEGIN_C_DECLS /* * Local Component structures */ PRTE_MODULE_EXPORT extern prte_state_base_component_t prte_mca_state_dvm_component; PRTE_EXPORT extern prte_state_base_module_t prte_state_dvm_module; END_C_DECLS #endif /* MCA_STATE_DVM_EXPORT_H */ prrte-4.1.1~rc2/src/mca/state/dvm/state_dvm.c0000664000175000017500000012173415203542132021236 0ustar alastairalastair/* * Copyright (c) 2015-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_fd.h" #include "src/util/nidmap.h" #include "src/util/pmix_os_dirpath.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/filem/filem.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/odls_types.h" #include "src/mca/plm/base/base.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/rml/rml.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/mca/state/base/base.h" #include "state_dvm.h" /* * Module functions: Global */ static int init(void); static int finalize(void); /* local functions */ static void init_complete(int fd, short args, void *cbdata); static void vm_ready(int fd, short args, void *cbata); static void check_complete(int fd, short args, void *cbdata); static void cleanup_job(int fd, short args, void *cbdata); static void job_started(int fd, short args, void *cbata); static void ready_for_debug(int fd, short args, void *cbata); /****************** * DVM module - used when mpirun is persistent ******************/ prte_state_base_module_t prte_state_dvm_module = { .init = init, .finalize = finalize, .activate_job_state = prte_state_base_activate_job_state, .add_job_state = prte_state_base_add_job_state, .set_job_state_callback = prte_state_base_set_job_state_callback, .remove_job_state = prte_state_base_remove_job_state, .activate_proc_state = prte_state_base_activate_proc_state, .add_proc_state = prte_state_base_add_proc_state, .set_proc_state_callback = prte_state_base_set_proc_state_callback, .remove_proc_state = prte_state_base_remove_proc_state }; static void dvm_notify(int sd, short args, void *cbdata); /* defined default state machine sequence - individual * plm's must add a state for launching daemons */ static prte_job_state_t launch_states[] = { PRTE_JOB_STATE_INIT, PRTE_JOB_STATE_INIT_COMPLETE, PRTE_JOB_STATE_ALLOCATE, PRTE_JOB_STATE_ALLOCATION_COMPLETE, PRTE_JOB_STATE_DAEMONS_LAUNCHED, PRTE_JOB_STATE_DAEMONS_REPORTED, PRTE_JOB_STATE_VM_READY, PRTE_JOB_STATE_MAP, PRTE_JOB_STATE_MAP_COMPLETE, PRTE_JOB_STATE_SYSTEM_PREP, PRTE_JOB_STATE_LAUNCH_APPS, PRTE_JOB_STATE_SEND_LAUNCH_MSG, PRTE_JOB_STATE_STARTED, PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE, PRTE_JOB_STATE_READY_FOR_DEBUG, PRTE_JOB_STATE_RUNNING, PRTE_JOB_STATE_REGISTERED, /* termination states */ PRTE_JOB_STATE_TERMINATED, PRTE_JOB_STATE_NOTIFY_COMPLETED, PRTE_JOB_STATE_NOTIFIED, PRTE_JOB_STATE_ALL_JOBS_COMPLETE }; static prte_state_cbfunc_t launch_callbacks[] = { prte_plm_base_setup_job, init_complete, prte_ras_base_allocate, prte_plm_base_allocation_complete, prte_plm_base_daemons_launched, prte_plm_base_daemons_reported, vm_ready, prte_rmaps_base_map_job, prte_plm_base_mapping_complete, prte_plm_base_complete_setup, prte_plm_base_launch_apps, prte_plm_base_send_launch_msg, job_started, prte_state_base_local_launch_complete, ready_for_debug, prte_plm_base_post_launch, prte_plm_base_registered, check_complete, dvm_notify, cleanup_job, prte_quit }; static prte_proc_state_t proc_states[] = { PRTE_PROC_STATE_RUNNING, PRTE_PROC_STATE_READY_FOR_DEBUG, PRTE_PROC_STATE_REGISTERED, PRTE_PROC_STATE_IOF_COMPLETE, PRTE_PROC_STATE_WAITPID_FIRED, PRTE_PROC_STATE_TERMINATED }; static prte_state_cbfunc_t proc_callbacks[] = { prte_state_base_track_procs, prte_state_base_track_procs, prte_state_base_track_procs, prte_state_base_track_procs, prte_state_base_track_procs, prte_state_base_track_procs }; static void force_quit(int fd, short args, void *cbdata) { PRTE_HIDE_UNUSED_PARAMS(fd, args); prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; /* give us a chance to stop the orteds */ prte_plm.terminate_orteds(); PMIX_RELEASE(caddy); } /************************ * Local variables ************************/ static bool terminate_dvm = false; static bool dvm_terminated = false; /************************ * API Definitions ************************/ static int init(void) { int i, rc; int num_states; /* setup the state machines */ PMIX_CONSTRUCT(&prte_job_states, pmix_list_t); PMIX_CONSTRUCT(&prte_proc_states, pmix_list_t); /* setup the job state machine */ num_states = sizeof(launch_states) / sizeof(prte_job_state_t); for (i = 0; i < num_states; i++) { if (PRTE_SUCCESS != (rc = prte_state.add_job_state(launch_states[i], launch_callbacks[i]))) { PRTE_ERROR_LOG(rc); } } /* add the termination response */ rc = prte_state.add_job_state(PRTE_JOB_STATE_DAEMONS_TERMINATED, prte_quit); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* add a default error response */ rc = prte_state.add_job_state(PRTE_JOB_STATE_FORCED_EXIT, force_quit); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* add callback to report progress, if requested */ rc = prte_state.add_job_state(PRTE_JOB_STATE_REPORT_PROGRESS, prte_state_base_report_progress); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) { prte_state_base_print_job_state_machine(); } /* populate the proc state machine to allow us to * track proc lifecycle changes */ num_states = sizeof(proc_states) / sizeof(prte_proc_state_t); for (i = 0; i < num_states; i++) { rc = prte_state.add_proc_state(proc_states[i], proc_callbacks[i]); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } } if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) { prte_state_base_print_proc_state_machine(); } return PRTE_SUCCESS; } static int finalize(void) { /* cleanup the state machines */ PMIX_LIST_DESTRUCT(&prte_proc_states); PMIX_LIST_DESTRUCT(&prte_job_states); return PRTE_SUCCESS; } static void files_ready(int status, void *cbdata) { prte_job_t *jdata = (prte_job_t *) cbdata; if (PRTE_SUCCESS != status) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); } else { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP); } } static void init_complete(int sd, short args, void *cbdata) { PRTE_HIDE_UNUSED_PARAMS(sd, args); prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PMIX_ACQUIRE_OBJECT(caddy); /* need to go thru allocate step in case someone wants to * expand the DVM */ PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_ALLOCATE); PMIX_RELEASE(caddy); } static void vm_ready(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; int rc, i; pmix_data_buffer_t buf; prte_job_t *jptr; prte_proc_t *dmn; int32_t v; pmix_value_t *val; pmix_status_t ret; PRTE_HIDE_UNUSED_PARAMS(fd, args, cbdata); PMIX_ACQUIRE_OBJECT(caddy); /* if this is my job, then we are done */ if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_LAUNCHED_DAEMONS, NULL, PMIX_BOOL)) { /* if there is more than one daemon in the job, then there * is just a little bit to do */ if (!prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL) && 1 < prte_process_info.num_daemons) { /* send the daemon map to every daemon in this DVM - we * do this here so we don't have to do it for every * job we are going to launch */ PMIX_DATA_BUFFER_CONSTRUCT(&buf); rc = prte_util_nidmap_create(prte_node_pool, &buf); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buf); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } /* get wireup info for daemons */ jptr = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); for (v = 0; v < jptr->procs->size; v++) { if (NULL == (dmn = (prte_proc_t *) pmix_pointer_array_get_item(jptr->procs, v))) { continue; } val = NULL; if (PMIX_SUCCESS != (ret = PMIx_Get(&dmn->name, PMIX_PROC_URI, NULL, 0, &val)) || NULL == val) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } rc = PMIx_Data_pack(NULL, &buf, &dmn->name, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } rc = PMIx_Data_pack(NULL, &buf, &val->data.string, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(ret); PMIX_DATA_BUFFER_DESTRUCT(&buf); PMIX_VALUE_RELEASE(val); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } PMIX_VALUE_RELEASE(val); } /* goes to all daemons */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_WIREUP, &buf))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buf); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_FORCED_EXIT); return; } PMIX_DATA_BUFFER_DESTRUCT(&buf); } } if (PMIX_CHECK_NSPACE(PRTE_PROC_MY_NAME->nspace, caddy->jdata->nspace)) { prte_dvm_ready = true; /* notify that the vm is ready */ if (0 > prte_state_base.parent_fd) { if (prte_state_base.ready_msg && prte_persistent) { fprintf(stdout, "DVM ready\n"); fflush(stdout); } } else { char ok = 'K'; pmix_fd_write(prte_state_base.parent_fd, 1, &ok); close(prte_state_base.parent_fd); prte_state_base.parent_fd = -1; } for (i = 0; i < prte_cache->size; i++) { jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_cache, i); if (NULL != jptr) { pmix_pointer_array_set_item(prte_cache, i, NULL); prte_plm.spawn(jptr); } } /* progress the job */ caddy->jdata->state = PRTE_JOB_STATE_VM_READY; PMIX_RELEASE(caddy); return; } /* position any required files */ if (PRTE_SUCCESS != prte_filem.preposition_files(caddy->jdata, files_ready, caddy->jdata)) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); } PMIX_RELEASE(caddy); } static void job_started(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata = caddy->jdata; pmix_info_t *iptr; time_t timestamp; pmix_proc_t *nptr; PRTE_HIDE_UNUSED_PARAMS(fd, args); /* if there is an originator for this job, notify them * that the first process of the job has been started */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DVM_JOB, NULL, PMIX_BOOL)) { /* dvm job => launch was requested by a TOOL, so we notify the launch proxy * and NOT the originator (as that would be us) */ nptr = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC) || NULL == nptr) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return; } timestamp = time(NULL); PMIX_INFO_CREATE(iptr, 5); /* target this notification solely to that one tool */ PMIX_INFO_LOAD(&iptr[0], PMIX_EVENT_CUSTOM_RANGE, nptr, PMIX_PROC); PMIX_PROC_RELEASE(nptr); /* pass the nspace of the spawned job */ PMIX_INFO_LOAD(&iptr[1], PMIX_NSPACE, jdata->nspace, PMIX_STRING); /* not to be delivered to a default event handler */ PMIX_INFO_LOAD(&iptr[2], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); /* provide the timestamp */ PMIX_INFO_LOAD(&iptr[3], PMIX_EVENT_TIMESTAMP, ×tamp, PMIX_TIME); PMIX_INFO_LOAD(&iptr[4], "prte.notify.donotloop", NULL, PMIX_BOOL); PMIx_Notify_event(PMIX_EVENT_JOB_START, &prte_process_info.myproc, PMIX_RANGE_CUSTOM, iptr, 5, NULL, NULL); PMIX_INFO_FREE(iptr, 5); } PMIX_RELEASE(caddy); } static void ready_for_debug(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata = caddy->jdata; pmix_proc_t *nptr; time_t timestamp; pmix_info_t *iptr; size_t ninfo; pmix_data_array_t darray; void *tinfo; pmix_status_t rc; int n; char *name; prte_app_context_t *app; PRTE_HIDE_UNUSED_PARAMS(fd, args); /* launch was requested by a TOOL, so we notify the launch proxy * and NOT the originator (as that would be us) */ nptr = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC) || NULL == nptr) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); goto DONE; } timestamp = time(NULL); PMIX_INFO_LIST_START(tinfo); /* target this notification solely to that one tool */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_CUSTOM_RANGE, nptr, PMIX_PROC); PMIX_PROC_RELEASE(nptr); /* pass the nspace of the job */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_NSPACE, jdata->nspace, PMIX_STRING); for (n=0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } /* if pset name was assigned, pass it */ if (prte_get_attribute(&app->attributes, PRTE_APP_PSET_NAME, (void**) &name, PMIX_STRING)) { PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_PSET_NAME, name, PMIX_STRING); free(name); } /* pass the argv from each app */ name = PMIx_Argv_join(app->argv, ' '); PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_APP_ARGV, name, PMIX_STRING); free(name); } /* not to be delivered to a default event handler */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); /* provide the timestamp */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_TIMESTAMP, ×tamp, PMIX_TIME); PMIX_INFO_LIST_ADD(rc, tinfo, "prte.notify.donotloop", NULL, PMIX_BOOL); PMIX_INFO_LIST_CONVERT(rc, tinfo, &darray); if (PMIX_ERR_EMPTY == rc) { iptr = NULL; ninfo = 0; } else if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PRTE_UPDATE_EXIT_STATUS(rc); PMIX_INFO_LIST_RELEASE(tinfo); PMIX_PROC_RELEASE(nptr); goto DONE; } else { iptr = (pmix_info_t *) darray.array; ninfo = darray.size; } PMIX_INFO_LIST_RELEASE(tinfo); PMIx_Notify_event(PMIX_READY_FOR_DEBUG, PRTE_PROC_MY_NAME, PMIX_RANGE_CUSTOM, iptr, ninfo, NULL, NULL); PMIX_INFO_FREE(iptr, ninfo); DONE: PMIX_RELEASE(caddy); } static void opcbfunc(pmix_status_t status, void *cbdata) { prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata; PMIX_POST_OBJECT(lk); lk->status = prte_pmix_convert_status(status); PRTE_PMIX_WAKEUP_THREAD(lk); } static void lkcbfunc(pmix_status_t status, void *cbdata) { prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata; PMIX_POST_OBJECT(lk); lk->status = prte_pmix_convert_status(status); PRTE_PMIX_WAKEUP_THREAD(lk); } static void check_complete(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_session_t *session; prte_job_t *jdata, *jptr; prte_proc_t *proc; int i, rc, nprocs; prte_node_t *node; prte_job_map_t *map; int32_t index; pmix_proc_t pname; prte_pmix_lock_t lock; uint8_t command = PRTE_PMIX_PURGE_PROC_CMD; pmix_data_buffer_t *buf; pmix_pointer_array_t procs; prte_timer_t *timer; prte_app_context_t *app; hwloc_obj_t obj; hwloc_obj_type_t type; hwloc_cpuset_t boundcpus, tgt; bool takeall, sep, *sepptr = &sep; prte_pmix_server_pset_t *pst, *pst2; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); jdata = caddy->jdata; pmix_output_verbose(2, prte_state_base_framework.framework_output, "%s state:dvm:check_job_complete on job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == jdata) ? "NULL" : PRTE_JOBID_PRINT(jdata->nspace)); if (NULL != jdata && prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER)) { /* timer is an prte_timer_t object */ prte_event_evtimer_del(timer->ev); PMIX_RELEASE(timer); prte_remove_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT); } if (NULL == jdata || PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { /* just check to see if the daemons are complete */ PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:dvm:check_job_complete - received NULL job, checking daemons", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (0 == pmix_list_get_size(&prte_rml_base.children)) { /* orteds are done! */ PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s prteds complete - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (NULL == jdata) { jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); } PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); PMIX_RELEASE(caddy); prte_dvm_ready = false; return; } prte_plm.terminate_orteds(); PMIX_RELEASE(caddy); return; } /* mark the job as terminated, but don't override any * abnormal termination flags */ if (jdata->state < PRTE_JOB_STATE_UNTERMINATED) { jdata->state = PRTE_JOB_STATE_TERMINATED; } /* see if there was any problem */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, NULL, PMIX_POINTER)) { rc = prte_pmix_convert_rc(jdata->exit_code); /* or whether we got cancelled by the user */ } else if (prte_get_attribute(&jdata->attributes, PRTE_JOB_CANCELLED, NULL, PMIX_BOOL)) { rc = prte_pmix_convert_rc(PRTE_ERR_JOB_CANCELLED); } else { rc = prte_pmix_convert_rc(jdata->exit_code); } /* if would be rare, but a very fast terminating job could conceivably * reach here prior to the spawn requestor being notified of spawn */ rc = prte_plm_base_spawn_response(rc, jdata); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* cleanup any pending server ops */ PMIX_LOAD_PROCID(&pname, jdata->nspace, PMIX_RANK_WILDCARD); prte_pmix_server_clear(&pname); /* cleanup the local procs as these are gone */ for (i = 0; i < prte_local_children->size; i++) { if (NULL == (proc = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { continue; } /* if this child is part of the job... */ if (PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) { /* clear the entry in the local children */ pmix_pointer_array_set_item(prte_local_children, i, NULL); PMIX_RELEASE(proc); // maintain accounting } } /* tell the IOF that the job is complete */ if (NULL != prte_iof.complete) { prte_iof.complete(jdata); } /* tell the PMIx subsystem the job is complete */ PRTE_PMIX_CONSTRUCT_LOCK(&lock); PMIx_server_deregister_nspace(pname.nspace, opcbfunc, &lock); PRTE_PMIX_WAIT_THREAD(&lock); PRTE_PMIX_DESTRUCT_LOCK(&lock); if (!prte_persistent) { /* update our exit status */ PRTE_UPDATE_EXIT_STATUS(jdata->exit_code); /* if this is an abnormal termination, report it */ if (jdata->state > PRTE_JOB_STATE_ERROR) { char *msg; msg = prte_dump_aborted_procs(jdata); if (NULL != msg) { pmix_byte_object_t bo; PMIX_BYTE_OBJECT_CONSTRUCT(&bo); bo.bytes = (char *) msg; bo.size = strlen(msg); PRTE_PMIX_CONSTRUCT_LOCK(&lock); rc = PMIx_server_IOF_deliver(&prte_process_info.myproc, PMIX_FWD_STDDIAG_CHANNEL, &bo, NULL, 0, lkcbfunc, (void *) &lock); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } else { /* wait for completion */ PRTE_PMIX_WAIT_THREAD(&lock); if (PMIX_SUCCESS != lock.status) { PMIX_ERROR_LOG(lock.status); } } PRTE_PMIX_DESTRUCT_LOCK(&lock); free(msg); } } /* if all of the jobs we are running are done, then shut us down */ for (i = 0; i < prte_job_data->size; i++) { jptr = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, i); if (NULL == jptr) { continue; } /* skip the daemon job */ if (PMIX_CHECK_NSPACE(jptr->nspace, PRTE_PROC_MY_NAME->nspace)) { continue; } if (jptr->state < PRTE_JOB_STATE_TERMINATED) { /* still alive - finish processing this job's termination */ goto release; } } /* Let the tools know that a job terminated before we shutdown */ if (jdata->state != PRTE_JOB_STATE_NOTIFIED) { PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm:check_job_completed state is terminated - activating notify", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); terminate_dvm = true; // flag that the DVM is to terminate PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFY_COMPLETED); PMIX_RELEASE(caddy); return; } /* if we fell thru to this point, then nobody is still * alive except the daemons, so just shut us down */ prte_plm.terminate_orteds(); PMIX_RELEASE(caddy); return; } if (NULL != prte_data_server_uri) { /* tell the data server to purge any data from this nspace */ PMIX_DATA_BUFFER_CREATE(buf); /* room number is ignored, but has to be included for pack sequencing */ i = 0; rc = PMIx_Data_pack(NULL, buf, &i, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); goto release; } rc = PMIx_Data_pack(NULL, buf, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); goto release; } /* pack the nspace to be purged */ pname.rank = PMIX_RANK_WILDCARD; rc = PMIx_Data_pack(NULL, buf, &pname, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); goto release; } /* send it to the data server */ PRTE_RML_SEND(rc, PRTE_PROC_MY_NAME->rank, buf, PRTE_RML_TAG_DATA_SERVER); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); } } release: /* Release the resources used by this job. Since some errmgrs may want * to continue using resources allocated to the job as part of their * fault recovery procedure, we only do this once the job is "complete". * Note that an aborted/killed job -is- flagged as complete and will * therefore have its resources released. We need to do this after * we call the errmgr so that any attempt to restart the job will * avoid doing so in the exact same place as the current job */ session = jdata->session; if(NULL != session){ for(i = 0; i < session->jobs->size; i++){ if(NULL != (jptr = pmix_pointer_array_get_item(session->jobs, i))){ if(PMIX_CHECK_NSPACE(jdata->nspace, jptr->nspace)){ pmix_pointer_array_set_item(session->jobs, i, NULL); break; } } } } if (NULL != jdata->map) { map = jdata->map; takeall = false; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { type = HWLOC_OBJ_PU; } else { type = HWLOC_OBJ_CORE; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16) || PRTE_MAPPING_BYUSER == PRTE_GET_MAPPING_POLICY(map->mapping) || PRTE_MAPPING_SEQ == PRTE_GET_MAPPING_POLICY(map->mapping)) { takeall = true; } boundcpus = hwloc_bitmap_alloc(); for (index = 0; index < map->nodes->size; index++) { node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, index); if (NULL == node) { continue; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm releasing procs from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); for (i = 0; i < node->procs->size; i++) { proc = (prte_proc_t *) pmix_pointer_array_get_item(node->procs, i); if (NULL == proc) { continue; } if (!PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) { /* skip procs from another job */ continue; } app = (prte_app_context_t*) pmix_pointer_array_get_item(jdata->apps, proc->app_idx); if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL) && !PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL)) { node->slots_inuse--; node->num_procs--; node->next_node_rank--; } /* release the resources held by the proc - only the first * cpu in the proc's cpuset was used to mark usage */ if (NULL != proc->cpuset) { if (0 != (rc = hwloc_bitmap_list_sscanf(boundcpus, proc->cpuset))) { pmix_output(0, "hwloc_bitmap_sscanf returned %s for the string %s", prte_strerror(rc), proc->cpuset); continue; } if (takeall) { tgt = boundcpus; } else { /* we only want to restore the first CPU of whatever region * the proc was bound to, so we have to first narrow the * bitmap down to only that region */ hwloc_bitmap_andnot(prte_rmaps_base.available, boundcpus, node->available); /* the set bits in the result are the bound cpus that are still * marked as in-use */ obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, type, 0); if (NULL == obj) { pmix_output(0, "COULD NOT GET BOUND CPU FOR RESOURCE RELEASE"); continue; } tgt = obj->cpuset; } hwloc_bitmap_or(node->available, node->available, tgt); } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm releasing proc %s from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proc->name), node->name)); /* set the entry in the node array to NULL */ pmix_pointer_array_set_item(node->procs, i, NULL); /* release the proc once for the map entry */ PMIX_RELEASE(proc); } /* set the node location to NULL */ pmix_pointer_array_set_item(map->nodes, index, NULL); /* maintain accounting */ PMIX_RELEASE(node); /* flag that the node is no longer in a map */ PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } hwloc_bitmap_free(boundcpus); PMIX_RELEASE(map); jdata->map = NULL; } // if this job has apps that named a pset, then remove them PMIX_LIST_FOREACH_SAFE(pst, pst2, &prte_pmix_server_globals.psets, prte_pmix_server_pset_t) { if (pst->jdata == jdata) { pmix_list_remove_item(&prte_pmix_server_globals.psets, &pst->super); PMIX_RELEASE(pst); } } /* if requested, check fd status for leaks */ if (prte_state_base.run_fdcheck) { prte_state_base_check_fds(jdata); } /* if this job started child jobs, then we need to abort all of its * child jobs that might still be running unless designated to * run independently of their parent */ if (0 < pmix_list_get_size(&jdata->children)) { PMIX_CONSTRUCT(&procs, pmix_pointer_array_t); pmix_pointer_array_init(&procs, 1, INT_MAX, 1); nprocs = 0; PMIX_LIST_FOREACH(jptr, &jdata->children, prte_job_t) { if (prte_get_attribute(&jptr->attributes, PRTE_JOB_CHILD_SEP, (void**)&sepptr, PMIX_BOOL) && !sep) { proc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&proc->name, jptr->nspace, PMIX_RANK_WILDCARD); pmix_pointer_array_add(&procs, proc); ++nprocs; if (1 == nprocs) { // output a warning message that at least one child is being terminated pmix_show_help("help-state-base.txt", "child-term", true, jdata->nspace, jptr->nspace); } } } if (0 < nprocs) { prte_plm.terminate_procs(&procs); for (i = 0; i < procs.size; i++) { if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(&procs, i))) { PMIX_RELEASE(proc); } } } PMIX_DESTRUCT(&procs); } if (jdata->state != PRTE_JOB_STATE_NOTIFIED) { PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm:check_job_completed state is terminated - activating notify", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFY_COMPLETED); /* mark the job as notified */ jdata->state = PRTE_JOB_STATE_NOTIFIED; } PMIX_POST_OBJECT(jdata); PMIX_RELEASE(caddy); } static void cleanup_job(int sd, short args, void *cbdata) { pmix_proc_t *nptr; prte_job_t *parent; prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(sd, args); PMIX_ACQUIRE_OBJECT(caddy); if (terminate_dvm && !dvm_terminated) { dvm_terminated = true; prte_plm.terminate_orteds(); } if (NULL != caddy->jdata) { /* if the job had a spawn parent remove it from the parents child list */ if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC)) { if(NULL != (parent = prte_get_job_data_object(nptr->nspace)) && !PMIX_CHECK_NSPACE(parent->nspace, PRTE_PROC_MY_NAME->nspace)){ pmix_list_remove_item(&parent->children, &caddy->jdata->super); /* We retained the jdata before adding it to the list - maintain ref count */ PMIX_RELEASE(caddy->jdata); } PMIX_PROC_RELEASE(nptr); } PMIX_RELEASE(caddy->jdata); } PMIX_RELEASE(caddy); } static void dvm_notify(int sd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata = caddy->jdata; prte_proc_t *pptr = NULL; int rc; pmix_data_buffer_t *reply; prte_daemon_cmd_flag_t command; bool notify = true, flag; pmix_proc_t *proc, pnotify; pmix_info_t *info; size_t ninfo; pmix_proc_t pname; pmix_data_buffer_t pbkt; pmix_data_range_t range = PMIX_RANGE_SESSION; pmix_status_t code, ret; char *errmsg = NULL; PRTE_HIDE_UNUSED_PARAMS(sd, args); PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm:dvm_notify called", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* see if there was any problem */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ABORTED_PROC, (void **) &pptr, PMIX_POINTER) && NULL != pptr) { rc = jdata->exit_code; /* or whether we got cancelled by the user */ } else if (prte_get_attribute(&jdata->attributes, PRTE_JOB_CANCELLED, NULL, PMIX_BOOL)) { rc = PRTE_ERR_JOB_CANCELLED; } else { rc = jdata->exit_code; } if (0 == rc && prte_get_attribute(&jdata->attributes, PRTE_JOB_SILENT_TERMINATION, NULL, PMIX_BOOL)) { notify = false; } /* if the jobid matches that of the requestor, then don't notify */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &proc, PMIX_PROC)) { if (PMIX_CHECK_NSPACE(proc->nspace, jdata->nspace)) { notify = false; } PMIX_PROC_RELEASE(proc); } if (notify) { PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm:dvm_notify notification requested", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* if it was an abnormal termination, then construct an appropriate * error message */ if (PRTE_SUCCESS != rc) { errmsg = prte_dump_aborted_procs(jdata); } /* construct the info to be provided */ if (NULL == errmsg) { ninfo = 3; } else { ninfo = 4; } PMIX_INFO_CREATE(info, ninfo); /* ensure this only goes to the job terminated event handler */ flag = true; PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, &flag, PMIX_BOOL); /* provide the status */ PMIX_INFO_LOAD(&info[1], PMIX_JOB_TERM_STATUS, &rc, PMIX_STATUS); /* tell the requestor which job or proc */ PMIX_LOAD_NSPACE(pname.nspace, jdata->nspace); if (NULL != pptr) { pname.rank = pptr->name.rank; } else { pname.rank = PMIX_RANK_WILDCARD; } PMIX_INFO_LOAD(&info[2], PMIX_EVENT_AFFECTED_PROC, &pname, PMIX_PROC); if (NULL != errmsg) { PMIX_INFO_LOAD(&info[3], PMIX_EVENT_TEXT_MESSAGE, errmsg, PMIX_STRING); free(errmsg); } /* pack the info for sending */ PMIX_DATA_BUFFER_CONSTRUCT(&pbkt); /* pack the status code */ code = PMIX_EVENT_JOB_END; if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &code, 1, PMIX_STATUS))) { PMIX_ERROR_LOG(ret); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_RELEASE(caddy); return; } /* pack the source - it cannot be me as that will cause * the pmix server to upcall the event back to me */ PMIX_LOAD_PROCID(&pnotify, jdata->nspace, 0); if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &pnotify, 1, PMIX_PROC))) { PMIX_ERROR_LOG(ret); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_RELEASE(caddy); return; } /* pack the range */ if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &range, 1, PMIX_DATA_RANGE))) { PMIX_ERROR_LOG(ret); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_RELEASE(caddy); return; } /* pack the number of infos */ if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, &ninfo, 1, PMIX_SIZE))) { PMIX_ERROR_LOG(ret); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_RELEASE(caddy); return; } /* pack the infos themselves */ if (PMIX_SUCCESS != (ret = PMIx_Data_pack(NULL, &pbkt, info, ninfo, PMIX_INFO))) { PMIX_ERROR_LOG(ret); PMIX_INFO_FREE(info, ninfo); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_RELEASE(caddy); return; } PMIX_INFO_FREE(info, ninfo); /* insert into pmix_data_buffer_t */ PMIX_DATA_BUFFER_CREATE(reply); /* we need to add a flag indicating this came from an invalid proc so that we will * inject it into our own PMIx server library */ rc = PMIx_Data_pack(NULL, reply, &PRTE_NAME_INVALID->rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(caddy); return; } rc = PMIx_Data_copy_payload(reply, &pbkt); PMIX_DATA_BUFFER_DESTRUCT(&pbkt); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(caddy); return; } /* we have to send the notification to all daemons so that * anyone watching for it can receive it */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_NOTIFICATION, reply))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); PMIX_RELEASE(caddy); return; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:dvm:dvm_notify notification sent", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_DATA_BUFFER_RELEASE(reply); } if (prte_persistent) { /* now ensure that _all_ daemons know that this job has terminated so even * those that did not participate in it will know to cleanup the resources * they assigned to the job. This is necessary now that the mapping function * has been moved to the backend daemons - otherwise, non-participating daemons * retain the slot assignments on the participating daemons, and then incorrectly * map subsequent jobs thinking those nodes are still "busy" */ PMIX_DATA_BUFFER_CREATE(reply); command = PRTE_DAEMON_DVM_CLEANUP_JOB_CMD; rc = PMIx_Data_pack(NULL, reply, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return; } rc = PMIx_Data_pack(NULL, reply, &jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(reply); return; } prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, reply); PMIX_DATA_BUFFER_RELEASE(reply); } // We are done with our use of job data and have notified the other daemons if (notify) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFIED); } PMIX_RELEASE(caddy); } prrte-4.1.1~rc2/src/mca/state/dvm/Makefile.am0000664000175000017500000000214215203542132021127 0ustar alastairalastair# # Copyright (c) 2015-2020 Intel, Inc. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ state_dvm.h \ state_dvm_component.c \ state_dvm.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_state_dvm_DSO component_noinst = component_install = prte_mca_state_dvm.la else component_noinst = libprtemca_state_dvm.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_state_dvm_la_SOURCES = $(sources) prte_mca_state_dvm_la_LDFLAGS = -module -avoid-version prte_mca_state_dvm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_state_dvm_la_SOURCES =$(sources) libprtemca_state_dvm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/state/base/0000775000175000017500000000000015203542132017220 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/state/base/owner.txt0000664000175000017500000000023715203542132021115 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: active prrte-4.1.1~rc2/src/mca/state/base/state_base_select.c0000664000175000017500000000323615203542132023041 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2015 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/util/pmix_output.h" #include "src/mca/state/base/base.h" int prte_state_base_select(void) { int exit_status = PRTE_SUCCESS; prte_state_base_component_t *best_component = NULL; prte_state_base_module_t *best_module = NULL; /* * Select the best component */ exit_status = pmix_mca_base_select("state", prte_state_base_framework.framework_output, &prte_state_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL); if (PRTE_SUCCESS != exit_status || NULL == best_module) { /* This will only happen if no component was selected */ exit_status = PRTE_ERROR; goto cleanup; } /* Save the winner */ prte_state = *best_module; /* Initialize the winner */ if (PRTE_SUCCESS != prte_state.init()) { exit_status = PRTE_ERROR; goto cleanup; } cleanup: return exit_status; } prrte-4.1.1~rc2/src/mca/state/base/state_base_frame.c0000664000175000017500000001317515203542132022657 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include #ifdef HAVE_UNISTD_H # include #endif #ifdef HAVE_SYS_TYPES_H # include #endif #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/class/pmix_list.h" #include "src/util/pmix_output.h" #include "src/mca/plm/plm_types.h" #include "src/runtime/prte_globals.h" #include "src/mca/state/base/base.h" #include "src/mca/state/base/static-components.h" /* * Globals */ prte_state_base_t prte_state_base = { .parent_fd = -1, .ready_msg = true, .run_fdcheck = false, .recoverable = false, .max_restarts = 0, .continuous = false }; prte_state_base_module_t prte_state = {0}; static int prte_state_base_register(pmix_mca_base_register_flag_t flags) { PRTE_HIDE_UNUSED_PARAMS(flags); prte_state_base.run_fdcheck = false; pmix_mca_base_var_register("prte", "state", "base", "check_fds", "Daemons should check fds for leaks after each job completes", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.run_fdcheck); prte_state_base.recoverable = false; pmix_mca_base_var_register("prte", "state", "base", "recoverable", "Default setting for recoverable runtime option", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.recoverable); prte_state_base.max_restarts = 0; pmix_mca_base_var_register("prte", "state", "base", "max_restarts", "Set default max number of times to restart a failed process", PMIX_MCA_BASE_VAR_TYPE_INT, &prte_state_base.max_restarts); prte_state_base.continuous = false; pmix_mca_base_var_register("prte", "state", "base", "continuous", "Set default policy for processes to run continuously until explicitly terminated", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.continuous); prte_state_base.error_non_zero_exit = true; pmix_mca_base_var_register("prte", "state", "base", "error_non_zero_exit", "Set default policy for marking it an error for a process to return a non-zero exit status", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.error_non_zero_exit); prte_state_base.show_launch_progress = false; pmix_mca_base_var_register("prte", "state", "base", "show_launch_progress", "Provide progress reports on DVM startup", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.show_launch_progress); prte_state_base.notifyerrors = false; pmix_mca_base_var_register("prte", "state", "base", "notify_errors", "Generate a PMIx event for reportable process errors", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.notifyerrors); prte_state_base.autorestart = false; pmix_mca_base_var_register("prte", "state", "base", "autorestart", "Automatically restart failed processes up to the max restart limit", PMIX_MCA_BASE_VAR_TYPE_BOOL, &prte_state_base.autorestart); return PRTE_SUCCESS; } static int prte_state_base_close(void) { /* Close selected component */ if (NULL != prte_state.finalize) { prte_state.finalize(); } return pmix_mca_base_framework_components_close(&prte_state_base_framework, NULL); } /** * * Function for finding and opening either all MCA components, or the one * * that was specifically requested via a MCA parameter. * */ static int prte_state_base_open(pmix_mca_base_open_flag_t flags) { /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_state_base_framework, flags); } PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, state, "PRTE State Machine", prte_state_base_register, prte_state_base_open, prte_state_base_close, prte_state_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_DEFAULT); static void prte_state_construct(prte_state_t *state) { state->job_state = PRTE_JOB_STATE_UNDEF; state->proc_state = PRTE_PROC_STATE_UNDEF; state->cbfunc = NULL; } PMIX_CLASS_INSTANCE(prte_state_t, pmix_list_item_t, prte_state_construct, NULL); static void prte_state_caddy_construct(prte_state_caddy_t *caddy) { memset(&caddy->ev, 0, sizeof(prte_event_t)); caddy->jdata = NULL; } static void prte_state_caddy_destruct(prte_state_caddy_t *caddy) { prte_event_del(&caddy->ev); if (NULL != caddy->jdata) { PMIX_RELEASE(caddy->jdata); } } PMIX_CLASS_INSTANCE(prte_state_caddy_t, pmix_object_t, prte_state_caddy_construct, prte_state_caddy_destruct); prrte-4.1.1~rc2/src/mca/state/base/base.h0000664000175000017500000000634515203542132020313 0ustar alastairalastair/* * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2018-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: */ #ifndef PRTE_MCA_STATE_BASE_H #define PRTE_MCA_STATE_BASE_H /* * includes */ #include "prte_config.h" #include "constants.h" #include "src/class/pmix_list.h" #include "src/util/pmix_printf.h" #include "src/mca/mca.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/mca/state/state.h" BEGIN_C_DECLS /** * Struct to hold data global to the state framework */ typedef struct { int parent_fd; bool ready_msg; bool run_fdcheck; bool recoverable; int max_restarts; bool continuous; bool error_non_zero_exit; bool show_launch_progress; bool notifyerrors; bool autorestart; } prte_state_base_t; PRTE_EXPORT extern prte_state_base_t prte_state_base; /* select a component */ PRTE_EXPORT int prte_state_base_select(void); /* debug tools */ PRTE_EXPORT void prte_state_base_print_job_state_machine(void); PRTE_EXPORT void prte_state_base_print_proc_state_machine(void); PRTE_EXPORT int prte_state_base_set_runtime_options(prte_job_t *jdata, char *spec); /* * Base functions */ PRTE_EXPORT void prte_state_base_activate_job_state(prte_job_t *jdata, prte_job_state_t state); PRTE_EXPORT int prte_state_base_add_job_state(prte_job_state_t state, prte_state_cbfunc_t cbfunc); PRTE_EXPORT int prte_state_base_set_job_state_callback(prte_job_state_t state, prte_state_cbfunc_t cbfunc); PRTE_EXPORT int prte_state_base_remove_job_state(prte_job_state_t state); PRTE_EXPORT void prte_util_print_job_state_machine(void); PRTE_EXPORT void prte_state_base_activate_proc_state(pmix_proc_t *proc, prte_proc_state_t state); PRTE_EXPORT int prte_state_base_add_proc_state(prte_proc_state_t state, prte_state_cbfunc_t cbfunc); PRTE_EXPORT int prte_state_base_set_proc_state_callback(prte_proc_state_t state, prte_state_cbfunc_t cbfunc); PRTE_EXPORT int prte_state_base_remove_proc_state(prte_proc_state_t state); PRTE_EXPORT void prte_util_print_proc_state_machine(void); /* common state processing functions */ PRTE_EXPORT void prte_state_base_local_launch_complete(int fd, short argc, void *cbdata); PRTE_EXPORT void prte_state_base_cleanup_job(int fd, short argc, void *cbdata); PRTE_EXPORT void prte_state_base_report_progress(int fd, short argc, void *cbdata); PRTE_EXPORT void prte_state_base_track_procs(int fd, short argc, void *cbdata); PRTE_EXPORT void prte_state_base_check_all_complete(int fd, short args, void *cbdata); PRTE_EXPORT void prte_state_base_check_fds(prte_job_t *jdata); PRTE_EXPORT void prte_state_base_notify_data_server(pmix_proc_t *target); // resource recovery PRTE_EXPORT void prte_state_base_recover_resources(prte_job_t *jdata, prte_proc_t *pptr); END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/state/base/help-state-base.txt0000664000175000017500000000514215203542132022741 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2024 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # [normal-termination-but] While %s job %s terminated normally, %d %s. Further examination may be required. # [bad-combination] The runtime option NOTIFYERRORS was set to true. This requests that the runtime provide a PMIx event whenever a job encounters an error - e.g., a process fails. The event is to be delivered to each remaining process in the job. However, this only can be done if the job is not to be terminated in the event of an error. The following runtime options govern that behavior: RECOVERABLE[=(bool)] if set to true, this indicates that the application wishes to consider the job as recoverable - i.e., the application is assuming responsibility for recovering from any process failure. This could include application-driven spawn of a substitute process or internal compensation for the missing process. This option defaults to a true value if the option is given without a value. CONTINUOUS[=(bool)] if set to true, this informs the runtime that the processes in this job are to run until explicitly terminated. Processes that fail are to be automatically restarted up to "max restarts" number of times. Notification of process failure is to be delivered to all processes in the application. This is the equivalent of specifying RECOVERABLE, NOTIFYERRORS, and AUTORESTART options except that the runtime, not the application, assumes responsibility for process recovery. This option defaults to a true value if the option is given without a value. You must specify one of the above in combination with NOTIFYERRORS in order to receive notifications of errors. Please correct the situation and try again. # [child-term] At least one child job is being terminated due to termination of its parent: Parent: %s Child: %s This behavior is controlled by setting the PMIX_SPAWN_CHILD_SEP attribute in the job info provided at time of spawn for the child job. When set to "true", the runtime will "separate" the child from its parent and allow it to continue execution after parent termination. Note that this is only true for parents that normally terminate - abnormal termination will always result in a complete teardown of all child jobs. In the absence of the attribute, the runtime will default to the "true" behavior. prrte-4.1.1~rc2/src/mca/state/base/state_base_options.c0000664000175000017500000004202615203542132023255 0ustar alastairalastair/* * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #include "prte_config.h" #include "constants.h" #if HAVE_UNISTD_H # include #endif #if HAVE_FCNTL_H # include #endif #include #include #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/plm/plm.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/prte_cmd_line.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/state/base/base.h" /* this function is called if pmix_server_dyn receives a * PMIX_RUNTIME_OPTIONS info struct */ int prte_state_base_set_runtime_options(prte_job_t *jdata, char *spec) { char **options, *ptr; int n; bool flag, *fptr = &flag; int32_t i32; prte_job_t *djob; prte_app_context_t *app; pmix_info_t info; pmix_value_t value; if (NULL == spec) { /* set everything to the defaults if not already set. We don't want to * have to check the value of BOOL settings everywhere we use them, so * we translate them here by removing the attribute if it is set to false, * and leaving it if it is set to true. If it isn't present, then it wasn't * provided via PMIx_Spawn and we instead set it based on the defaults */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT); } } else { /* set it based on default value */ if (prte_state_base.error_non_zero_exit) { prte_set_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS); } } else { /* set it based on default value */ if (prte_state_base.show_launch_progress) { prte_set_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE); } } else { /* set it based on default value */ if (prte_state_base.recoverable) { prte_set_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS); } } else { /* set it based on default value */ if (prte_state_base.continuous) { prte_set_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS); } } else { /* set it based on default value */ if (prte_state_base.notifyerrors) { prte_set_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_AUTORESTART, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_AUTORESTART); } } else { /* set it based on default value */ if (prte_state_base.autorestart) { prte_set_attribute(&jdata->attributes, PRTE_JOB_AUTORESTART, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_EXEC_AGENT, NULL, PMIX_STRING)) { if (NULL != prte_odls_globals.exec_agent) { prte_set_attribute(&jdata->attributes, PRTE_JOB_EXEC_AGENT, PRTE_ATTR_GLOBAL, prte_odls_globals.exec_agent, PMIX_STRING); } } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT, (void**)&fptr, PMIX_BOOL)) { /* it is present - check the value */ if (!flag) { /* remove the attribute */ prte_remove_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT); } } else { /* set it based on default value */ if (prte_fwd_environment) { prte_set_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } } /* check the apps for max restarts */ if (0 < prte_state_base.max_restarts) { for (n = 0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } if (!prte_get_attribute(&app->attributes, PRTE_APP_MAX_RESTARTS, NULL, PMIX_INT32)) { prte_set_attribute(&app->attributes, PRTE_APP_MAX_RESTARTS, PRTE_ATTR_GLOBAL, &prte_state_base.max_restarts, PMIX_INT32); } } } } else { options = PMIx_Argv_split(spec, ','); for (n=0; NULL != options[n]; n++) { /* see if there is an '=' */ ptr = strchr(options[n], '='); if (NULL != ptr) { *ptr = '\0'; ++ptr; if ('\0' == *ptr) { /* missing the value */ pmix_show_help("help-prte-rmaps-base.txt", "missing-value", true, "runtime options", options[n], "empty"); PMIx_Argv_free(options); return PRTE_ERR_BAD_PARAM; } } PMIX_VALUE_LOAD(&value, ptr, PMIX_STRING); // just in case we need to evaluate a bool /* check the options */ if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_ERROR_NZ)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_NOLAUNCH)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); /* if we are not in a persistent DVM, then make sure we also * apply this to the daemons */ if (!prte_persistent) { djob = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); prte_set_attribute(&djob->attributes, PRTE_JOB_DO_NOT_LAUNCH, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_NOSPAWN)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_SPAWN, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); /* if we are not in a persistent DVM, then make sure we also * apply this to the daemons */ if (!prte_persistent) { djob = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); prte_set_attribute(&djob->attributes, PRTE_JOB_DO_NOT_SPAWN, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_SHOW_PROGRESS)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_NOTIFY_ERRORS)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_RECOVERABLE)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_AUTORESTART)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_AUTORESTART, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_CONTINUOUS)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_MAX_RESTARTS)) { if (NULL == ptr || '\0' == *ptr) { /* missing the value */ pmix_show_help("help-prte-rmaps-base.txt", "missing-value", true, "runtime options", options[n], "empty"); PMIx_Argv_free(options); return PRTE_ERR_BAD_PARAM; } i32 = strtol(ptr, NULL, 10); for (n = 0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } prte_set_attribute(&app->attributes, PRTE_APP_MAX_RESTARTS, PRTE_ATTR_GLOBAL, &i32, PMIX_INT32); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_EXEC_AGENT)) { prte_set_attribute(&jdata->attributes, PRTE_JOB_EXEC_AGENT, PRTE_ATTR_GLOBAL, ptr, PMIX_STRING); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_DEFAULT_EXEC_AGENT)) { prte_remove_attribute(&jdata->attributes, PRTE_JOB_EXEC_AGENT); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_STOP_ON_EXEC)) { flag = PMIX_CHECK_TRUE(&value); if (flag) { prte_set_attribute(&jdata->attributes, PRTE_JOB_STOP_ON_EXEC, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } else { prte_remove_attribute(&jdata->attributes, PRTE_JOB_STOP_ON_EXEC); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_STOP_IN_INIT)) { flag = PMIX_CHECK_TRUE(&value); if (flag) { prte_set_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_INIT, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); /* also must add to job-level cache */ PMIX_INFO_LOAD(&info, PMIX_DEBUG_STOP_IN_INIT, NULL, PMIX_BOOL); pmix_server_cache_job_info(jdata, &info); } else { prte_remove_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_INIT); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_STOP_IN_APP)) { flag = PMIX_CHECK_TRUE(&value); if (flag) { prte_set_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_APP, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); /* also must add to job-level cache */ PMIX_INFO_LOAD(&info, PMIX_DEBUG_STOP_IN_APP, NULL, PMIX_BOOL); pmix_server_cache_job_info(jdata, &info); } else { prte_remove_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_APP); } } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_TIMEOUT)) { n = PMIX_CONVERT_TIME(ptr); prte_set_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT, PRTE_ATTR_GLOBAL, &n, PMIX_INT); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_SPAWN_TIMEOUT)) { n = PMIX_CONVERT_TIME(ptr); prte_set_attribute(&jdata->attributes, PRTE_SPAWN_TIMEOUT, PRTE_ATTR_GLOBAL, &n, PMIX_INT); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_STACK_TRACES)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_STACKTRACES, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_REPORT_STATE)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_REPORT_STATE, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_AGG_HELP)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_NOAGG_HELP, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_OUTPUT_PROCTABLE)) { if (NULL == ptr || '\0' == *ptr) { /* no value provided, so assume stdout */ ptr = "-"; } prte_set_attribute(&jdata->attributes, PRTE_JOB_OUTPUT_PROCTABLE, PRTE_ATTR_GLOBAL, ptr, PMIX_STRING); } else if (PMIX_CHECK_CLI_OPTION(options[n], PRTE_CLI_FWD_ENVIRON)) { flag = PMIX_CHECK_TRUE(&value); prte_set_attribute(&jdata->attributes, PRTE_JOB_FWD_ENVIRONMENT, PRTE_ATTR_GLOBAL, &flag, PMIX_BOOL); } else { pmix_show_help("help-prte-rmaps-base.txt", "unrecognized-policy", true, "runtime options", spec); return PRTE_ERR_SILENT; } } PMIx_Argv_free(options); } /* if notify-error is set but neither recovery nor continuous were specified, * then notifications will not be given as we will terminate the job upon * error. Detect that situation, provide a show-help explaining the problem, * and then error out */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_NOTIFY_ERRORS, NULL, PMIX_BOOL) && !prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL) && !prte_get_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, NULL, PMIX_BOOL)) { pmix_show_help("help-state-base.txt", "bad-combination", true); return PRTE_ERR_SILENT; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/state/base/state_base_fns.c0000664000175000017500000011702615203542132022353 0ustar alastairalastair/* * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 IBM Corporation. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file **/ #include "prte_config.h" #include "constants.h" #if HAVE_UNISTD_H # include #endif #if HAVE_FCNTL_H # include #endif #include #include #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/pmix/pmix-internal.h" #include "src/util/pmix_argv.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/grpcomm/grpcomm.h" #include "src/mca/iof/base/base.h" #include "src/mca/plm/plm.h" #include "src/mca/rmaps/base/base.h" #include "src/rml/rml.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/state/base/base.h" void prte_state_base_activate_job_state(prte_job_t *jdata, prte_job_state_t state) { pmix_list_item_t *itm, *any = NULL, *error = NULL; prte_state_t *s; prte_state_caddy_t *caddy; for (itm = pmix_list_get_first(&prte_job_states); itm != pmix_list_get_end(&prte_job_states); itm = pmix_list_get_next(itm)) { s = (prte_state_t *) itm; if (s->job_state == PRTE_JOB_STATE_ANY) { /* save this place */ any = itm; } if (s->job_state == PRTE_JOB_STATE_ERROR) { error = itm; } if (s->job_state == state) { PRTE_REACHING_JOB_STATE(jdata, state); if (NULL == s->cbfunc) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "%s NULL CBFUNC FOR JOB %s STATE %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == jdata) ? "ALL" : PRTE_JOBID_PRINT(jdata->nspace), prte_job_state_to_str(state))); return; } caddy = PMIX_NEW(prte_state_caddy_t); if (NULL != jdata) { caddy->jdata = jdata; caddy->job_state = state; PMIX_RETAIN(jdata); } PRTE_PMIX_THREADSHIFT(caddy, prte_event_base, s->cbfunc); return; } } /* if we get here, then the state wasn't found, so execute * the default handler if it is defined */ if (PRTE_JOB_STATE_ERROR < state && NULL != error) { s = (prte_state_t *) error; } else if (NULL != any) { s = (prte_state_t *) any; } else { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "ACTIVATE: JOB STATE %s NOT REGISTERED", prte_job_state_to_str(state))); return; } if (NULL == s->cbfunc) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "ACTIVATE: ANY STATE HANDLER NOT DEFINED")); return; } caddy = PMIX_NEW(prte_state_caddy_t); if (NULL != jdata) { caddy->jdata = jdata; caddy->job_state = state; PMIX_RETAIN(jdata); } PRTE_REACHING_JOB_STATE(jdata, state); PRTE_PMIX_THREADSHIFT(caddy, prte_event_base, s->cbfunc); } int prte_state_base_add_job_state(prte_job_state_t state, prte_state_cbfunc_t cbfunc) { prte_state_t *st; /* check for uniqueness */ PMIX_LIST_FOREACH(st, &prte_job_states, prte_state_t) { if (st->job_state == state) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "DUPLICATE STATE DEFINED: %s", prte_job_state_to_str(state))); return PRTE_ERR_BAD_PARAM; } } st = PMIX_NEW(prte_state_t); st->job_state = state; st->cbfunc = cbfunc; pmix_list_append(&prte_job_states, &(st->super)); return PRTE_SUCCESS; } int prte_state_base_set_job_state_callback(prte_job_state_t state, prte_state_cbfunc_t cbfunc) { pmix_list_item_t *item; prte_state_t *st; for (item = pmix_list_get_first(&prte_job_states); item != pmix_list_get_end(&prte_job_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; if (st->job_state == state) { st->cbfunc = cbfunc; return PRTE_SUCCESS; } } /* if not found, assume SYS priority and install it */ st = PMIX_NEW(prte_state_t); st->job_state = state; st->cbfunc = cbfunc; pmix_list_append(&prte_job_states, &(st->super)); return PRTE_SUCCESS; } int prte_state_base_remove_job_state(prte_job_state_t state) { pmix_list_item_t *item; prte_state_t *st; for (item = pmix_list_get_first(&prte_job_states); item != pmix_list_get_end(&prte_job_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; if (st->job_state == state) { pmix_list_remove_item(&prte_job_states, item); PMIX_RELEASE(item); return PRTE_SUCCESS; } } return PRTE_ERR_NOT_FOUND; } void prte_state_base_print_job_state_machine(void) { pmix_list_item_t *item; prte_state_t *st; pmix_output(0, "PRTE_JOB_STATE_MACHINE:"); for (item = pmix_list_get_first(&prte_job_states); item != pmix_list_get_end(&prte_job_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; pmix_output(0, "\tState: %s cbfunc: %s", prte_job_state_to_str(st->job_state), (NULL == st->cbfunc) ? "NULL" : "DEFINED"); } } /**** PROC STATE MACHINE ****/ void prte_state_base_activate_proc_state(pmix_proc_t *proc, prte_proc_state_t state) { pmix_list_item_t *itm, *any = NULL, *error = NULL; prte_state_t *s; prte_state_caddy_t *caddy; for (itm = pmix_list_get_first(&prte_proc_states); itm != pmix_list_get_end(&prte_proc_states); itm = pmix_list_get_next(itm)) { s = (prte_state_t *) itm; if (s->proc_state == PRTE_PROC_STATE_ANY) { /* save this place */ any = itm; } if (s->proc_state == PRTE_PROC_STATE_ERROR) { error = itm; } if (s->proc_state == state) { PRTE_REACHING_PROC_STATE(proc, state); if (NULL == s->cbfunc) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "%s NULL CBFUNC FOR PROC %s STATE %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state))); return; } caddy = PMIX_NEW(prte_state_caddy_t); caddy->name = *proc; caddy->proc_state = state; PRTE_PMIX_THREADSHIFT(caddy, prte_event_base, s->cbfunc); return; } } /* if we get here, then the state wasn't found, so execute * the default handler if it is defined */ if (PRTE_PROC_STATE_ERROR < state && NULL != error) { s = (prte_state_t *) error; } else if (NULL != any) { s = (prte_state_t *) any; } else { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "INCREMENT: ANY STATE NOT FOUND")); return; } if (NULL == s->cbfunc) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "ACTIVATE: ANY STATE HANDLER NOT DEFINED")); return; } caddy = PMIX_NEW(prte_state_caddy_t); caddy->name = *proc; caddy->proc_state = state; PRTE_REACHING_PROC_STATE(proc, state); PRTE_PMIX_THREADSHIFT(caddy, prte_event_base, s->cbfunc); } int prte_state_base_add_proc_state(prte_proc_state_t state, prte_state_cbfunc_t cbfunc) { pmix_list_item_t *item; prte_state_t *st; /* check for uniqueness */ for (item = pmix_list_get_first(&prte_proc_states); item != pmix_list_get_end(&prte_proc_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; if (st->proc_state == state) { PMIX_OUTPUT_VERBOSE((1, prte_state_base_framework.framework_output, "DUPLICATE STATE DEFINED: %s", prte_proc_state_to_str(state))); return PRTE_ERR_BAD_PARAM; } } st = PMIX_NEW(prte_state_t); st->proc_state = state; st->cbfunc = cbfunc; pmix_list_append(&prte_proc_states, &(st->super)); return PRTE_SUCCESS; } int prte_state_base_set_proc_state_callback(prte_proc_state_t state, prte_state_cbfunc_t cbfunc) { pmix_list_item_t *item; prte_state_t *st; for (item = pmix_list_get_first(&prte_proc_states); item != pmix_list_get_end(&prte_proc_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; if (st->proc_state == state) { st->cbfunc = cbfunc; return PRTE_SUCCESS; } } return PRTE_ERR_NOT_FOUND; } int prte_state_base_remove_proc_state(prte_proc_state_t state) { pmix_list_item_t *item; prte_state_t *st; for (item = pmix_list_get_first(&prte_proc_states); item != pmix_list_get_end(&prte_proc_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; if (st->proc_state == state) { pmix_list_remove_item(&prte_proc_states, item); PMIX_RELEASE(item); return PRTE_SUCCESS; } } return PRTE_ERR_NOT_FOUND; } void prte_state_base_print_proc_state_machine(void) { pmix_list_item_t *item; prte_state_t *st; pmix_output(0, "PRTE_PROC_STATE_MACHINE:"); for (item = pmix_list_get_first(&prte_proc_states); item != pmix_list_get_end(&prte_proc_states); item = pmix_list_get_next(item)) { st = (prte_state_t *) item; pmix_output(0, "\tState: %s cbfunc: %s", prte_proc_state_to_str(st->proc_state), (NULL == st->cbfunc) ? "NULL" : "DEFINED"); } } void prte_state_base_local_launch_complete(int fd, short argc, void *cbdata) { prte_state_caddy_t *state = (prte_state_caddy_t *) cbdata; prte_job_t *jdata = state->jdata; bool found = false; PRTE_HIDE_UNUSED_PARAMS(fd, argc); found = prte_get_attribute(&jdata->attributes, PRTE_JOB_SHOW_PROGRESS, NULL, PMIX_BOOL); if (found) { if (0 == jdata->num_daemons_reported % 100 || jdata->num_daemons_reported == prte_process_info.num_daemons) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_REPORT_PROGRESS); } } PMIX_RELEASE(state); } void prte_state_base_cleanup_job(int fd, short argc, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; PRTE_HIDE_UNUSED_PARAMS(fd, argc); PMIX_ACQUIRE_OBJECT(caddy); jdata = caddy->jdata; PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:base:cleanup on job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == jdata) ? "NULL" : PRTE_JOBID_PRINT(jdata->nspace))); /* flag that we were notified */ jdata->state = PRTE_JOB_STATE_NOTIFIED; /* send us back thru job complete */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); PMIX_RELEASE(caddy); } void prte_state_base_report_progress(int fd, short argc, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; PRTE_HIDE_UNUSED_PARAMS(fd, argc); PMIX_ACQUIRE_OBJECT(caddy); jdata = caddy->jdata; pmix_output(prte_clean_output, "App launch reported: %d (out of %d) daemons - %d (out of %d) procs", (int) jdata->num_daemons_reported, (int) prte_process_info.num_daemons, (int) jdata->num_launched, (int) jdata->num_procs); PMIX_RELEASE(caddy); } void prte_state_base_notify_data_server(pmix_proc_t *target) { pmix_data_buffer_t *buf; int rc, room = -1; uint8_t cmd = PRTE_PMIX_PURGE_PROC_CMD; /* if nobody local to us published anything, then we can ignore this */ if (PMIX_NSPACE_INVALID(prte_pmix_server_globals.server.nspace)) { return; } PMIX_DATA_BUFFER_CREATE(buf); /* pack the room number */ rc = PMIx_Data_pack(NULL, buf, &room, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return; } /* load the command */ rc = PMIx_Data_pack(NULL, buf, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return; } /* provide the target */ rc = PMIx_Data_pack(NULL, buf, target, 1, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(buf); return; } /* send the request to the server */ PRTE_RML_SEND(rc, prte_pmix_server_globals.server.rank, buf, PRTE_RML_TAG_DATA_SERVER); if (PRTE_SUCCESS != rc) { PMIX_DATA_BUFFER_RELEASE(buf); } } static void opcbfunc(pmix_status_t status, void *cbdata) { prte_pmix_lock_t *lock = (prte_pmix_lock_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(status); PRTE_PMIX_WAKEUP_THREAD(lock); } void prte_state_base_track_procs(int fd, short argc, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; pmix_proc_t *proc; prte_proc_state_t state; prte_job_t *jdata; prte_proc_t *pdata; int i; pmix_proc_t target; pmix_rank_t threshold; PRTE_HIDE_UNUSED_PARAMS(fd, argc); PMIX_ACQUIRE_OBJECT(caddy); proc = &caddy->name; state = caddy->proc_state; pmix_output_verbose(5, prte_state_base_framework.framework_output, "%s state:base:track_procs called for proc %s state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)); /* get the job object for this proc */ if (NULL == (jdata = prte_get_job_data_object(proc->nspace))) { goto cleanup; } if (PRTE_PROC_STATE_READY_FOR_DEBUG == state) { if (prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_ON_EXEC, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_INIT, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_APP, NULL, PMIX_BOOL)) { if (PRTE_PROC_IS_MASTER) { threshold = jdata->num_procs; } else { threshold = jdata->num_local_procs; } if (PMIX_RANK_LOCAL_PEERS == proc->rank) { jdata->num_ready_for_debug += jdata->num_local_procs; } else { jdata->num_ready_for_debug++; } if (jdata->num_ready_for_debug < threshold) { goto cleanup; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:base all local %s procs on node %s ready for debug", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), proc->nspace, prte_process_info.nodename)); /* let the DVM master know we are ready */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_READY_FOR_DEBUG); } goto cleanup; } pdata = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank); if (NULL == pdata) { goto cleanup; } if (PRTE_PROC_STATE_RUNNING == state) { /* update the proc state */ if (pdata->state < PRTE_PROC_STATE_TERMINATED) { pdata->state = state; } jdata->num_launched++; if (1 == jdata->num_launched) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_STARTED); } if (jdata->num_launched == jdata->num_procs) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_RUNNING); } } else if (PRTE_PROC_STATE_REGISTERED == state) { /* update the proc state */ if (pdata->state < PRTE_PROC_STATE_TERMINATED) { pdata->state = state; } jdata->num_reported++; if (jdata->num_reported == jdata->num_procs) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_REGISTERED); } } else if (PRTE_PROC_STATE_IOF_COMPLETE == state) { /* update the proc state */ if (pdata->state < PRTE_PROC_STATE_TERMINATED) { pdata->state = state; } /* Release the IOF file descriptors */ if (NULL != prte_iof.close) { prte_iof.close(proc, PRTE_IOF_STDALL); } PRTE_FLAG_SET(pdata, PRTE_PROC_FLAG_IOF_COMPLETE); if (PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_WAITPID)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } } else if (PRTE_PROC_STATE_WAITPID_FIRED == state) { /* update the proc state */ if (pdata->state < PRTE_PROC_STATE_TERMINATED) { pdata->state = state; } PRTE_FLAG_SET(pdata, PRTE_PROC_FLAG_WAITPID); if (PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_IOF_COMPLETE)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } } else if (PRTE_PROC_STATE_TERMINATED == state) { if (pdata->state == state) { pmix_output_verbose(5, prte_state_base_framework.framework_output, "%s state:base:track_procs proc %s already in state %s. Skip transition.", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)); goto cleanup; } /* update the proc state */ PRTE_FLAG_UNSET(pdata, PRTE_PROC_FLAG_ALIVE); if (pdata->state < PRTE_PROC_STATE_TERMINATED) { pdata->state = state; } if (PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_LOCAL)) { PMIx_server_deregister_client(proc, NULL, NULL); } /* if we are trying to terminate and our routes are * gone, then terminate ourselves IF no local procs * remain (might be some from another job) */ if (prte_prteds_term_ordered && 0 == pmix_list_get_size(&prte_rml_base.children)) { for (i = 0; i < prte_local_children->size; i++) { pdata = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL != pdata && PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_ALIVE)) { /* at least one is still alive */ goto cleanup; } } /* call our appropriate exit procedure */ pmix_output_verbose(5, prte_state_base_framework.framework_output, "%s state:base all routes and children gone - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME)); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); goto cleanup; } /* track job status */ jdata->num_terminated++; if (jdata->num_terminated == jdata->num_procs) { /* if requested, check fd status for leaks */ if (prte_state_base.run_fdcheck) { prte_state_base_check_fds(jdata); } /* if ompi-server is around, then notify it to purge * any session-related info */ if (NULL != prte_data_server_uri) { PMIX_LOAD_PROCID(&target, jdata->nspace, PMIX_RANK_WILDCARD); prte_state_base_notify_data_server(&target); } PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); } } cleanup: PMIX_RELEASE(caddy); } void prte_state_base_check_all_complete(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_proc_t *proc; int i; int32_t j; prte_job_t *job; prte_node_t *node; prte_job_map_t *map; int32_t index; bool one_still_alive, flag; pmix_rank_t lowest = 0; int32_t i32, *i32ptr; prte_pmix_lock_t lock; prte_app_context_t *app; prte_pmix_server_pset_t *pst, *pst2; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); jdata = caddy->jdata; pmix_output_verbose(2, prte_state_base_framework.framework_output, "%s state:base:check_job_complete on job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == jdata) ? "NULL" : PRTE_JOBID_PRINT(jdata->nspace)); if (NULL == jdata || PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { /* just check to see if the daemons are complete */ PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:base:check_job_complete - received NULL job, checking daemons", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); goto CHECK_DAEMONS; } else { /* mark the job as terminated, but don't override any * abnormal termination flags */ if (jdata->state < PRTE_JOB_STATE_UNTERMINATED) { jdata->state = PRTE_JOB_STATE_TERMINATED; } } /* tell the IOF that the job is complete */ if (NULL != prte_iof.complete) { prte_iof.complete(jdata); } /* tell the PMIx server to release its data */ PRTE_PMIX_CONSTRUCT_LOCK(&lock); PMIx_server_deregister_nspace(jdata->nspace, opcbfunc, &lock); PRTE_PMIX_WAIT_THREAD(&lock); PRTE_PMIX_DESTRUCT_LOCK(&lock); i32ptr = &i32; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_NUM_NONZERO_EXIT, (void **) &i32ptr, PMIX_INT32)) { flag = prte_get_attribute(&jdata->attributes, PRTE_JOB_ERROR_NONZERO_EXIT, NULL, PMIX_BOOL); if (flag) { if (!prte_report_child_jobs_separately || 1 == PRTE_LOCAL_JOBID(jdata->nspace)) { /* update the exit code */ PRTE_UPDATE_EXIT_STATUS(lowest); } /* warn user */ pmix_show_help("help-state-base.txt", "normal-termination-but", true, (1 == PRTE_LOCAL_JOBID(jdata->nspace)) ? "the primary" : "child", (1 == PRTE_LOCAL_JOBID(jdata->nspace)) ? "" : PRTE_LOCAL_JOBID_PRINT(jdata->nspace), i32, (1 == i32) ? "process returned\na non-zero exit code." : "processes returned\nnon-zero exit codes."); } } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed declared job %s terminated with state " "%s - checking all jobs", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), prte_job_state_to_str(jdata->state))); /* if this job is a continuously operating one or recoverable, then don't do * anything further - just return here */ if (NULL != jdata && (prte_get_attribute(&jdata->attributes, PRTE_JOB_CONTINUOUS, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL))) { goto CHECK_ALIVE; } /* if the job that is being checked is the HNP, then we are * trying to terminate the orteds. In that situation, we * do -not- check all jobs - we simply notify the HNP * that the orteds are complete. Also check special case * if jdata is NULL - we want * to definitely declare the job done if the orteds * have completed, no matter what else may be happening. * This can happen if a ctrl-c hits in the "wrong" place * while launching */ CHECK_DAEMONS: if (jdata == NULL || PMIX_CHECK_NSPACE(jdata->nspace, PRTE_PROC_MY_NAME->nspace)) { if (0 == pmix_list_get_size(&prte_rml_base.children)) { /* orteds are done! */ PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s orteds complete - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (NULL == jdata) { jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); } PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); PMIX_RELEASE(caddy); return; } PMIX_RELEASE(caddy); return; } /* Release the resources used by this job. Since some errmgrs may want * to continue using resources allocated to the job as part of their * fault recovery procedure, we only do this once the job is "complete". * Note that an aborted/killed job -is- flagged as complete and will * therefore have its resources released. We need to do this after * we call the errmgr so that any attempt to restart the job will * avoid doing so in the exact same place as the current job */ if (NULL != jdata->map && jdata->state == PRTE_JOB_STATE_TERMINATED) { map = jdata->map; for (index = 0; index < map->nodes->size; index++) { node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, index); if (NULL == node) { continue; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s releasing procs for job %s from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), node->name)); for (i = 0; i < node->procs->size; i++) { proc = (prte_proc_t *) pmix_pointer_array_get_item(node->procs, i); if (NULL == proc) { continue; } app = (prte_app_context_t*) pmix_pointer_array_get_item(jdata->apps, proc->app_idx); if (!PMIX_CHECK_NSPACE(proc->name.nspace, jdata->nspace)) { /* skip procs from another job */ continue; } if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL) && !PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL)) { node->slots_inuse--; node->num_procs--; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s releasing proc %s from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&proc->name), node->name)); /* set the entry in the node array to NULL */ pmix_pointer_array_set_item(node->procs, i, NULL); /* release the proc once for the map entry */ PMIX_RELEASE(proc); } /* set the node location to NULL */ pmix_pointer_array_set_item(map->nodes, index, NULL); /* maintain accounting */ PMIX_RELEASE(node); } PMIX_RELEASE(map); jdata->map = NULL; } // if this job has apps that named a pset, then remove them PMIX_LIST_FOREACH_SAFE(pst, pst2, &prte_pmix_server_globals.psets, prte_pmix_server_pset_t) { if (pst->jdata == jdata) { pmix_list_remove_item(&prte_pmix_server_globals.psets, &pst->super); PMIX_RELEASE(pst); } } CHECK_ALIVE: /* now check to see if all jobs are done - trigger notification of this jdata * object when we find it */ one_still_alive = false; for (j = 0; j < prte_job_data->size; j++) { job = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, j); if (NULL == job) { continue; } /* skip the daemon job */ if (PMIX_CHECK_NSPACE(job->nspace, PRTE_PROC_MY_NAME->nspace)) { continue; } /* if this is the job we are checking AND it normally terminated, * then activate the "notify_completed" state - this will release * the job state, but is provided so that the HNP main code can * take alternative actions if desired. If the state is killed_by_cmd, * then go ahead and release it. We cannot release it if it * abnormally terminated as mpirun needs the info so it can * report appropriately to the user * * NOTE: do not release the primary job (j=1) so we * can pretty-print completion message */ if (NULL != jdata && PMIX_CHECK_NSPACE(job->nspace, jdata->nspace)) { if (jdata->state == PRTE_JOB_STATE_TERMINATED) { PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed state is terminated - activating notify", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_NOTIFY_COMPLETED); one_still_alive = true; } else if (jdata->state == PRTE_JOB_STATE_KILLED_BY_CMD || jdata->state == PRTE_JOB_STATE_NOTIFIED) { PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed state is killed or notified - cleaning up", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* release this object, ensuring that the * pointer array internal accounting * is maintained! */ pmix_pointer_array_set_item(prte_job_data, j, NULL); PMIX_RELEASE(jdata); } continue; } /* if the job is flagged to not be monitored, skip it */ if (PRTE_FLAG_TEST(job, PRTE_JOB_FLAG_DO_NOT_MONITOR)) { continue; } /* when checking for job termination, we must be sure to NOT check * our own job as it - rather obviously - has NOT terminated! */ if (PRTE_JOB_STATE_NOTIFIED != job->state) { /* we have at least one job that is not done yet - we cannot * just return, though, as we need to ensure we cleanout the * job data for the job that just completed */ PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed job %s is not terminated (%d:%d)", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job->nspace), job->num_terminated, job->num_procs)); one_still_alive = true; } else { PMIX_OUTPUT_VERBOSE( (2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed job %s is terminated (%d vs %d [%s])", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(job->nspace), job->num_terminated, job->num_procs, (NULL == jdata) ? "UNKNOWN" : prte_job_state_to_str(jdata->state))); } } /* if a job is still alive, we just return */ if (one_still_alive) { PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed at least one job is not terminated", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_RELEASE(caddy); return; } /* if we get here, then all jobs are done, so terminate */ PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:base:check_job_completed all jobs terminated", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* stop the job timeout event, if set */ if (NULL != prte_mpiexec_timeout) { PMIX_RELEASE(prte_mpiexec_timeout); prte_mpiexec_timeout = NULL; } /* set the exit status to 0 - this will only happen if it * wasn't already set by an error condition */ PRTE_UPDATE_EXIT_STATUS(0); /* order daemon termination - this tells us to cleanup * our local procs as well as telling remote daemons * to die */ prte_plm.terminate_orteds(); PMIX_RELEASE(caddy); } void prte_state_base_check_fds(prte_job_t *jdata) { int nfds, i, fdflags, flflags; char path[1024], info[256], **list = NULL, *status, *result, *r2; ssize_t rc; struct flock fl; bool flk; int cnt = 0; /* get the number of available file descriptors * for this daemon */ nfds = getdtablesize(); result = NULL; /* loop over them and get their info */ for (i = 0; i < nfds; i++) { fdflags = fcntl(i, F_GETFD); if (-1 == fdflags) { /* no open fd in that slot */ continue; } flflags = fcntl(i, F_GETFL); if (-1 == flflags) { /* no open fd in that slot */ continue; } snprintf(path, 1024, "/proc/self/fd/%d", i); memset(info, 0, 256); /* read the info about this fd */ rc = readlink(path, info, 256); if (-1 == rc) { /* this fd is unavailable */ continue; } /* get any file locking status */ fl.l_type = F_WRLCK; fl.l_whence = 0; fl.l_start = 0; fl.l_len = 0; if (-1 == fcntl(i, F_GETLK, &fl)) { flk = false; } else { flk = true; } /* construct the list of capabilities */ if (fdflags & FD_CLOEXEC) { PMIx_Argv_append_nosize(&list, "cloexec"); } if (flflags & O_APPEND) { PMIx_Argv_append_nosize(&list, "append"); } if (flflags & O_NONBLOCK) { PMIx_Argv_append_nosize(&list, "nonblock"); } /* from the man page: * Unlike the other values that can be specified in flags, * the access mode values O_RDONLY, O_WRONLY, and O_RDWR, * do not specify individual bits. Rather, they define * the low order two bits of flags, and defined respectively * as 0, 1, and 2. */ if (O_RDONLY == (flflags & 3)) { PMIx_Argv_append_nosize(&list, "rdonly"); } else if (O_WRONLY == (flflags & 3)) { PMIx_Argv_append_nosize(&list, "wronly"); } else { PMIx_Argv_append_nosize(&list, "rdwr"); } if (flk && F_UNLCK != fl.l_type) { if (F_WRLCK == fl.l_type) { PMIx_Argv_append_nosize(&list, "wrlock"); } else { PMIx_Argv_append_nosize(&list, "rdlock"); } } if (NULL != list) { status = PMIx_Argv_join(list, ' '); PMIx_Argv_free(list); list = NULL; if (NULL == result) { pmix_asprintf(&result, " %d\t(%s)\t%s\n", i, info, status); } else { pmix_asprintf(&r2, "%s %d\t(%s)\t%s\n", result, i, info, status); free(result); result = r2; } free(status); } ++cnt; } pmix_asprintf(&r2, "%s: %d open file descriptors after job %d completed\n%s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), cnt, PRTE_LOCAL_JOBID(jdata->nspace), result); pmix_output(0, "%s", r2); free(result); free(r2); } void prte_state_base_recover_resources(prte_job_t *jdata, prte_proc_t *pptr) { prte_node_t *node, *nptr; prte_job_map_t *map; prte_proc_t *p; int n, node_idx, proc_idx, rc; bool takeall, empty; hwloc_obj_t obj; hwloc_obj_type_t type; hwloc_cpuset_t boundcpus, tgt; PRTE_HIDE_UNUSED_PARAMS(jdata, pptr); node = pptr->node; map = jdata->map; // recover node resources node->slots_inuse--; node->num_procs--; node->next_node_rank--; // find the node in the map node_idx = INT_MAX; for (n=0; n < jdata->map->nodes->size; n++) { nptr = (prte_node_t*)pmix_pointer_array_get_item(jdata->map->nodes, n); if (NULL == nptr) { continue; } if (nptr == node) { node_idx = n; } } // find the proc in the node array proc_idx = INT_MAX; for (n=0; n < node->procs->size; n++) { p = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, n); if (NULL == p) { continue; } if (p == pptr) { proc_idx = n; } } // determine how cpus were handled takeall = false; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_HWT_CPUS, NULL, PMIX_BOOL)) { type = HWLOC_OBJ_PU; } else { type = HWLOC_OBJ_CORE; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_PES_PER_PROC, NULL, PMIX_UINT16) || PRTE_MAPPING_BYUSER == PRTE_GET_MAPPING_POLICY(map->mapping) || PRTE_MAPPING_SEQ == PRTE_GET_MAPPING_POLICY(map->mapping)) { takeall = true; } boundcpus = hwloc_bitmap_alloc(); /* release the resources held by the proc - only the first * cpu in the proc's cpuset was used to mark usage */ if (NULL != pptr->cpuset) { if (0 != (rc = hwloc_bitmap_list_sscanf(boundcpus, pptr->cpuset))) { pmix_output(0, "hwloc_bitmap_sscanf returned %s for the string %s", prte_strerror(rc), pptr->cpuset); goto next; } if (takeall) { tgt = boundcpus; } else { /* we only want to restore the first CPU of whatever region * the proc was bound to, so we have to first narrow the * bitmap down to only that region */ hwloc_bitmap_andnot(prte_rmaps_base.available, boundcpus, node->available); /* the set bits in the result are the bound cpus that are still * marked as in-use */ obj = hwloc_get_obj_inside_cpuset_by_type(node->topology->topo, prte_rmaps_base.available, type, 0); if (NULL == obj) { pmix_output(0, "COULD NOT GET BOUND CPU FOR RESOURCE RELEASE"); goto next; } tgt = obj->cpuset; } hwloc_bitmap_or(node->available, node->available, tgt); } next: if (proc_idx < INT_MAX) { /* set the entry in the node's proc array to NULL */ pmix_pointer_array_set_item(node->procs, proc_idx, NULL); } /* release the proc once for the map entry */ PMIX_RELEASE(pptr); // if the node is empty for this job, then remove it from the map empty = true; for (n=0; n < node->procs->size; n++) { p = (prte_proc_t*)pmix_pointer_array_get_item(node->procs, n); if (NULL != p && PMIx_Check_nspace(p->name.nspace, jdata->nspace)) { empty = false; break; } } if (empty) { if (node_idx < INT_MAX) { /* set the node location to NULL */ pmix_pointer_array_set_item(map->nodes, node_idx, NULL); } /* maintain accounting */ PMIX_RELEASE(node); /* flag that the node is no longer in a map */ PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } // release the scratch bitmap hwloc_bitmap_free(boundcpus); } prrte-4.1.1~rc2/src/mca/state/base/Makefile.am0000664000175000017500000000137515203542132021262 0ustar alastairalastair# # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST += \ base/help-state-base.txt headers += \ base/base.h libprtemca_state_la_SOURCES += \ base/state_base_frame.c \ base/state_base_select.c \ base/state_base_fns.c \ base/state_base_options.c prrte-4.1.1~rc2/src/mca/state/state.h0000664000175000017500000003231615203542132017604 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2024 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /**** PRTE STATE MACHINE ****/ /* States are treated as events so that the event * library can sequence them. Each state consists * of an event, a job or process state, a pointer * to the respective object, and a callback function * to be executed for that state. Events can be defined * at different priorities - e.g., SYS priority for * events associated with launching jobs, and ERR priority * for events associated with abnormal termination of * a process. * * The state machine consists of a list of state objects, * each defining a state-cbfunc pair. At startup, a default * list is created by the base functions which is then * potentially customized by selected components within * the various PRTE frameworks. For example, a PLM component * may need to insert states in the launch procedure, or may * want to redirect a particular state callback to a custom * function. * * For convenience, an ANY state can be defined along with a generic * callback function, with the corresponding state object * placed at the end of the state machine. Setting the * machine to a state that has not been explicitly defined * will cause this default action to be executed. Thus, you * don't have to explicitly define a state-cbfunc pair * for every job or process state. */ #ifndef _PRTE_STATE_H_ #define _PRTE_STATE_H_ #include "prte_config.h" #include "src/class/pmix_list.h" #include "src/event/event-internal.h" #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/plm/plm_types.h" #include "src/mca/state/state_types.h" #include "src/runtime/prte_globals.h" #include "src/util/error_strings.h" BEGIN_C_DECLS /* * MCA Framework - put here to access the pmix_output channel * in the macros */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_state_base_framework; #ifdef HAVE_SYS_TIME_H # include #endif /* For ease in debugging the state machine, it is STRONGLY recommended * that the functions be accessed using the following macros */ /* Timestamp for state printouts */ #define PRTE_STATE_GET_TIMESTAMP(t) \ do { \ struct timeval tv; \ gettimeofday(&tv, NULL); \ t = tv.tv_sec; \ t += (double) tv.tv_usec / 1000000.0; \ } while (0); #define PRTE_ACTIVATE_JOB_STATE(j, s) \ do { \ prte_job_t *shadow = (j); \ if (prte_state_base_framework.framework_verbose > 0) { \ double timestamp = 0.0; \ PRTE_STATE_GET_TIMESTAMP(timestamp); \ pmix_output_verbose(1, prte_state_base_framework.framework_output, \ "%s [%f] ACTIVATE JOB %s STATE %s AT %s:%d", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), timestamp, \ (NULL == shadow) ? "NULL" : PRTE_JOBID_PRINT(shadow->nspace), \ prte_job_state_to_str((s)), __FILE__, __LINE__); \ } \ prte_state.activate_job_state(shadow, (s)); \ } while (0); #define PRTE_ACTIVATE_PROC_STATE(p, s) \ do { \ pmix_proc_t *shadow = (p); \ if (prte_state_base_framework.framework_verbose > 0) { \ double timestamp = 0.0; \ PRTE_STATE_GET_TIMESTAMP(timestamp); \ pmix_output_verbose(1, prte_state_base_framework.framework_output, \ "%s [%f] ACTIVATE PROC %s STATE %s AT %s:%d", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), timestamp, \ (NULL == shadow) ? "NULL" : PRTE_NAME_PRINT(shadow), \ prte_proc_state_to_str((s)), __FILE__, __LINE__); \ } \ prte_state.activate_proc_state(shadow, (s)); \ } while (0); /* Called when actually arriving (reaching) the state with priority k */ #define PRTE_REACHING_JOB_STATE(j, s) \ do { \ prte_job_t *shadow = (j); \ if (prte_state_base_framework.framework_verbose > 0) { \ double timestamp = 0.0; \ PRTE_STATE_GET_TIMESTAMP(timestamp); \ pmix_output_verbose(1, prte_state_base_framework.framework_output, \ "%s [%f] ACTIVATING JOB %s STATE %s", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), timestamp, \ (NULL == shadow) ? "NULL" : PRTE_JOBID_PRINT(shadow->nspace), \ prte_job_state_to_str((s))); \ } \ } while (0); #define PRTE_REACHING_PROC_STATE(p, s) \ do { \ pmix_proc_t *shadow = (p); \ if (prte_state_base_framework.framework_verbose > 0) { \ double timestamp = 0.0; \ PRTE_STATE_GET_TIMESTAMP(timestamp); \ pmix_output_verbose(1, prte_state_base_framework.framework_output, \ "%s [%f] ACTIVATING PROC %s STATE %s", \ PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), timestamp, \ (NULL == shadow) ? "NULL" : PRTE_NAME_PRINT(shadow), \ prte_proc_state_to_str((s))); \ } \ } while (0); /** * Module initialization function. * * @retval PRTE_SUCCESS The operation completed successfully * @retval PRTE_ERROR An unspecifed error occurred */ typedef int (*prte_state_base_module_init_fn_t)(void); /** * Module finalization function. * * @retval PRTE_SUCCESS The operation completed successfully * @retval PRTE_ERROR An unspecifed error occurred */ typedef int (*prte_state_base_module_finalize_fn_t)(void); /**** JOB STATE APIs ****/ /* Job states are accessed via prte_job_t objects as they are only * used in PRTE tools and not application processes. APIs are provided * for assembling and editing the state machine, as well as activating * a specific job state * * Note the inherent assumption in this design that any customization * of the state machine will at least start with the base states - i.e., * that one would start with the default machine and edit it to add, * remove, or modify callbacks as required. Alternatively, one could * just clear the list entirely and assemble a fully custom state * machine - both models are supported. */ /* Activate a state in the job state machine. * * Creates and activates an event with the callback corresponding to the * specified job state. If the specified state is not found: * * 1. if a state machine entry for PRTE_JOB_STATE_ERROR was given, and * the state is an error state (i.e., PRTE_JOB_STATE_ERROR <= state), * then the callback for the ERROR state will be used * * 2. if a state machine entry for PRTE_JOB_STATE_ANY was given, and * the state is not an error state (i.e., state < PRTE_JOB_STATE_ERROR), * then the callback for the ANY state will be used * * 3. if neither of the above is true, then the call will be ignored. */ typedef void (*prte_state_base_module_activate_job_state_fn_t)(prte_job_t *jdata, prte_job_state_t state); /* Add a state to the job state machine. * */ typedef int (*prte_state_base_module_add_job_state_fn_t)(prte_job_state_t state, prte_state_cbfunc_t cbfunc); /* Set the callback function for a state in the job state machine. * */ typedef int (*prte_state_base_module_set_job_state_callback_fn_t)(prte_job_state_t state, prte_state_cbfunc_t cbfunc); /* Remove a state from the job state machine. * */ typedef int (*prte_state_base_module_remove_job_state_fn_t)(prte_job_state_t state); /**** Proc STATE APIs ****/ /* Proc states are accessed via pmix_proc_t as the state machine * must be available to both application processes and PRTE tools. APIs are * providedfor assembling and editing the state machine, as well as activating * a specific proc state * * Note the inherent assumption in this design that any customization * of the state machine will at least start with the base states - i.e., * that one would start with the default machine and edit it to add, * remove, or modify callbacks as required. Alternatively, one could * just clear the list entirely and assemble a fully custom state * machine - both models are supported. */ /* Activate a proc state. * * Creates and activates an event with the callback corresponding to the * specified proc state. If the specified state is not found: * * 1. if a state machine entry for PRTE_PROC_STATE_ERROR was given, and * the state is an error state (i.e., PRTE_PROC_STATE_ERROR <= state), * then the callback for the ERROR state will be used * * 2. if a state machine entry for PRTE_PROC_STATE_ANY was given, and * the state is not an error state (i.e., state < PRTE_PROC_STATE_ERROR), * then the callback for the ANY state will be used * * 3. if neither of the above is true, then the call will be ignored. */ typedef void (*prte_state_base_module_activate_proc_state_fn_t)(pmix_proc_t *proc, prte_proc_state_t state); /* Add a state to the proc state machine. * */ typedef int (*prte_state_base_module_add_proc_state_fn_t)(prte_proc_state_t state, prte_state_cbfunc_t cbfunc); /* Set the callback function for a state in the proc state machine. * */ typedef int (*prte_state_base_module_set_proc_state_callback_fn_t)(prte_proc_state_t state, prte_state_cbfunc_t cbfunc); /* Remove a state from the proc state machine. * */ typedef int (*prte_state_base_module_remove_proc_state_fn_t)(prte_proc_state_t state); /* * Module Structure */ struct prte_state_base_module_1_0_0_t { /** Initialization Function */ prte_state_base_module_init_fn_t init; /** Finalization Function */ prte_state_base_module_finalize_fn_t finalize; /* Job state APIs */ prte_state_base_module_activate_job_state_fn_t activate_job_state; prte_state_base_module_add_job_state_fn_t add_job_state; prte_state_base_module_set_job_state_callback_fn_t set_job_state_callback; prte_state_base_module_remove_job_state_fn_t remove_job_state; /* Proc state APIs */ prte_state_base_module_activate_proc_state_fn_t activate_proc_state; prte_state_base_module_add_proc_state_fn_t add_proc_state; prte_state_base_module_set_proc_state_callback_fn_t set_proc_state_callback; prte_state_base_module_remove_proc_state_fn_t remove_proc_state; }; typedef struct prte_state_base_module_1_0_0_t prte_state_base_module_1_0_0_t; typedef prte_state_base_module_1_0_0_t prte_state_base_module_t; PRTE_EXPORT extern prte_state_base_module_t prte_state; /* * State Component */ typedef pmix_mca_base_component_t prte_state_base_component_t; /* * Macro for use in components that are of type state */ #define PRTE_STATE_BASE_VERSION_1_0_0 PRTE_MCA_BASE_VERSION_3_0_0("state", 1, 0, 0) END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/state/Makefile.am0000664000175000017500000000140215203542132020337 0ustar alastairalastair# # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_state.la libprtemca_state_la_SOURCES = # pkgdata setup EXTRA_DIST = # local files headers = state.h state_types.h libprtemca_state_la_SOURCES += $(headers) # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h prrte-4.1.1~rc2/src/mca/state/prted/0000775000175000017500000000000015203542132017424 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/state/prted/owner.txt0000664000175000017500000000023715203542132021321 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: active prrte-4.1.1~rc2/src/mca/state/prted/state_prted.h0000664000175000017500000000165215203542132022117 0ustar alastairalastair/* * Copyright (c) 2011 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * */ #ifndef MCA_STATE_PRTED_EXPORT_H #define MCA_STATE_PRTED_EXPORT_H #include "prte_config.h" #include "src/mca/state/state.h" BEGIN_C_DECLS /* * Local Component structures */ PRTE_MODULE_EXPORT extern prte_state_base_component_t prte_mca_state_prted_component; PRTE_EXPORT extern prte_state_base_module_t prte_state_prted_module; END_C_DECLS #endif /* MCA_STATE_PRTED_EXPORT_H */ prrte-4.1.1~rc2/src/mca/state/prted/state_prted_component.c0000664000175000017500000000455015203542132024174 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/util/pmix_output.h" #include "src/mca/state/base/base.h" #include "src/mca/state/state.h" #include "state_prted.h" /* * Public string for version number */ const char *prte_mca_state_prted_component_version_string = "PRTE STATE prted MCA component version " PRTE_VERSION; /* * Local functionality */ static int state_prted_open(void); static int state_prted_close(void); static int state_prted_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointer to our public functions in it */ prte_state_base_component_t prte_mca_state_prted_component = { PRTE_STATE_BASE_VERSION_1_0_0, /* Component name and version */ .pmix_mca_component_name = "prted", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = state_prted_open, .pmix_mca_close_component = state_prted_close, .pmix_mca_query_component = state_prted_component_query, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, state, prted) static int my_priority = 100; static int state_prted_open(void) { return PRTE_SUCCESS; } static int state_prted_close(void) { return PRTE_SUCCESS; } static int state_prted_component_query(pmix_mca_base_module_t **module, int *priority) { if (PRTE_PROC_IS_DAEMON) { /* set our priority high as we are the default for prteds */ *priority = my_priority; *module = (pmix_mca_base_module_t *) &prte_state_prted_module; return PRTE_SUCCESS; } *priority = -1; *module = NULL; return PRTE_ERROR; } prrte-4.1.1~rc2/src/mca/state/prted/state_prted.c0000664000175000017500000006552415203542132022122 0ustar alastairalastair/* * Copyright (c) 2011-2017 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020-2021 IBM Corporation. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #ifdef HAVE_UNISTD_H # include #endif /* HAVE_UNISTD_H */ #include #include "src/pmix/pmix-internal.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/rmaps/rmaps_types.h" #include "src/rml/rml.h" #include "src/prted/pmix/pmix_server_internal.h" #include "src/runtime/data_server/prte_data_server.h" #include "src/runtime/prte_quit.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_output.h" #include "src/util/proc_info.h" #include "src/util/session_dir.h" #include "src/mca/state/base/base.h" #include "state_prted.h" /* * Module functions: Global */ static int init(void); static int finalize(void); /****************** * PRTED module ******************/ prte_state_base_module_t prte_state_prted_module = { .init = init, .finalize = finalize, .activate_job_state = prte_state_base_activate_job_state, .add_job_state = prte_state_base_add_job_state, .set_job_state_callback = prte_state_base_set_job_state_callback, .remove_job_state = prte_state_base_remove_job_state, .activate_proc_state = prte_state_base_activate_proc_state, .add_proc_state = prte_state_base_add_proc_state, .set_proc_state_callback = prte_state_base_set_proc_state_callback, .remove_proc_state = prte_state_base_remove_proc_state }; /* Local functions */ static void track_jobs(int fd, short argc, void *cbdata); static void track_procs(int fd, short argc, void *cbdata); static int pack_state_update(pmix_data_buffer_t *buf, prte_job_t *jdata); /* defined default state machines */ static prte_job_state_t job_states[] = { PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE, PRTE_JOB_STATE_READY_FOR_DEBUG }; static prte_state_cbfunc_t job_callbacks[] = { track_jobs, track_jobs }; static prte_proc_state_t proc_states[] = { PRTE_PROC_STATE_RUNNING, PRTE_PROC_STATE_READY_FOR_DEBUG, PRTE_PROC_STATE_REGISTERED, PRTE_PROC_STATE_IOF_COMPLETE, PRTE_PROC_STATE_WAITPID_FIRED, PRTE_PROC_STATE_TERMINATED }; static prte_state_cbfunc_t proc_callbacks[] = { track_procs, track_procs, track_procs, track_procs, track_procs, track_procs }; /************************ * API Definitions ************************/ static int init(void) { int num_states, i, rc; /* setup the state machine */ PMIX_CONSTRUCT(&prte_job_states, pmix_list_t); PMIX_CONSTRUCT(&prte_proc_states, pmix_list_t); num_states = sizeof(job_states) / sizeof(prte_job_state_t); for (i = 0; i < num_states; i++) { rc = prte_state.add_job_state(job_states[i], job_callbacks[i]); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } } /* add a default error response */ rc = prte_state.add_job_state(PRTE_JOB_STATE_FORCED_EXIT, prte_quit); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* add a state for when we are ordered to terminate */ rc = prte_state.add_job_state(PRTE_JOB_STATE_DAEMONS_TERMINATED, prte_quit); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) { prte_state_base_print_job_state_machine(); } /* populate the proc state machine to allow us to * track proc lifecycle changes */ num_states = sizeof(proc_states) / sizeof(prte_proc_state_t); for (i = 0; i < num_states; i++) { rc = prte_state.add_proc_state(proc_states[i], proc_callbacks[i]); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } } if (5 < pmix_output_get_verbosity(prte_state_base_framework.framework_output)) { prte_state_base_print_proc_state_machine(); } return PRTE_SUCCESS; } static int finalize(void) { /* cleanup the state machines */ PMIX_LIST_DESTRUCT(&prte_proc_states); PMIX_LIST_DESTRUCT(&prte_job_states); return PRTE_SUCCESS; } static void track_jobs(int fd, short argc, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; pmix_data_buffer_t *alert = NULL; prte_plm_cmd_flag_t cmd; int rc, i; prte_proc_state_t running = PRTE_PROC_STATE_RUNNING; prte_proc_t *child; PRTE_HIDE_UNUSED_PARAMS(fd, argc); PMIX_ACQUIRE_OBJECT(caddy); switch (caddy->job_state) { case PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE: PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted:track_jobs sending local launch complete for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(caddy->jdata->nspace))); /* update the HNP with all proc states for this job */ PMIX_DATA_BUFFER_CREATE(alert); /* pack update state command */ cmd = PRTE_PLM_LOCAL_LAUNCH_COMP_CMD; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &caddy->jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } for (i = 0; i < prte_local_children->size; i++) { child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL == child) { continue; } /* if this child is part of the job... */ if (PMIX_CHECK_NSPACE(child->name.nspace, caddy->jdata->nspace)) { /* pack the child's vpid */ rc = PMIx_Data_pack(NULL, alert, &child->name.rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the child's pid */ rc = PMIx_Data_pack(NULL, alert, &child->pid, 1, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* If this proc failed to start, then send that info. * However if it normally terminated then do not send the info. * Instead report it as running here, and the child waitpid * function will send back the normal terminated state when the * the job is complete. */ if (PRTE_PROC_STATE_TERMINATED < child->state) { rc = PMIx_Data_pack(NULL, alert, &child->state, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack its exit code */ rc = PMIx_Data_pack(NULL, alert, &child->exit_code, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } } else { /* pack the RUNNING state to avoid any race conditions */ rc = PMIx_Data_pack(NULL, alert, &running, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } } } } break; case PRTE_JOB_STATE_READY_FOR_DEBUG: PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted:track_jobs sending ready for debug for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(caddy->jdata->nspace))); /* update the HNP with all proc states for this job */ PMIX_DATA_BUFFER_CREATE(alert); running = PRTE_PROC_STATE_READY_FOR_DEBUG; /* pack update state command */ cmd = PRTE_PLM_READY_FOR_DEBUG_CMD; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &caddy->jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } for (i = 0; i < prte_local_children->size; i++) { child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL == child) { continue; } /* if this child is part of the job... */ if (PMIX_CHECK_NSPACE(child->name.nspace, caddy->jdata->nspace)) { /* pack the child's vpid */ rc = PMIx_Data_pack(NULL, alert, &child->name.rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the child's PID */ rc = PMIx_Data_pack(NULL, alert, &child->pid, 1, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } } } break; default: break; } if (NULL != alert) { /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); } } cleanup: PMIX_RELEASE(caddy); } static void opcbfunc(pmix_status_t status, void *cbdata) { prte_pmix_lock_t *lk = (prte_pmix_lock_t *) cbdata; PMIX_POST_OBJECT(lk); lk->status = prte_pmix_convert_status(status); PRTE_PMIX_WAKEUP_THREAD(lk); } static void track_procs(int fd, short argc, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; pmix_proc_t *proc; prte_proc_state_t state; prte_job_t *jdata; prte_proc_t *pdata, *pptr; pmix_data_buffer_t *alert; int rc, i; prte_plm_cmd_flag_t cmd; int32_t index; prte_job_map_t *map; prte_node_t *node; pmix_proc_t target; prte_pmix_lock_t lock; prte_app_context_t *app; PRTE_HIDE_UNUSED_PARAMS(fd, argc); PMIX_ACQUIRE_OBJECT(caddy); proc = &caddy->name; state = caddy->proc_state; pmix_output_verbose(5, prte_state_base_framework.framework_output, "%s state:prted:track_procs called for proc %s state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(proc), prte_proc_state_to_str(state)); /* get the job object for this proc */ if (NULL == (jdata = prte_get_job_data_object(proc->nspace))) { goto cleanup; } if (PRTE_PROC_STATE_READY_FOR_DEBUG == state) { if (prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_ON_EXEC, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_INIT, NULL, PMIX_BOOL) || prte_get_attribute(&jdata->attributes, PRTE_JOB_STOP_IN_APP, NULL, PMIX_BOOL)) { if (PMIX_RANK_LOCAL_PEERS == proc->rank) { jdata->num_ready_for_debug += jdata->num_local_procs; } else { jdata->num_ready_for_debug++; } if (jdata->num_ready_for_debug < jdata->num_local_procs) { goto cleanup; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:prted all local %s procs on node %s ready for debug", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), proc->nspace, prte_process_info.nodename)); /* let the DVM master know we are ready */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_READY_FOR_DEBUG); } goto cleanup; } pdata = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, proc->rank); if (NULL == pdata) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); goto cleanup; } if (PRTE_PROC_STATE_RUNNING == state) { /* update the proc state */ pdata->state = state; jdata->num_launched++; if (jdata->num_launched == jdata->num_local_procs) { /* tell the state machine that all local procs for this job * were launched so that it can do whatever it needs to do, * like send a state update message for all procs to the HNP */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE); } /* don't update until we are told that all are done */ } else if (PRTE_PROC_STATE_REGISTERED == state) { /* update the proc state */ pdata->state = state; jdata->num_reported++; if (jdata->num_reported == jdata->num_local_procs) { /* once everyone registers, notify the HNP */ PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted: notifying HNP all local registered", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PMIX_DATA_BUFFER_CREATE(alert); /* pack registered command */ cmd = PRTE_PLM_REGISTERED_CMD; rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &proc->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack all the local child vpids */ for (i = 0; i < prte_local_children->size; i++) { pptr = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL == pptr) { continue; } if (PMIX_CHECK_NSPACE(pptr->name.nspace, proc->nspace)) { rc = PMIx_Data_pack(NULL, alert, &pptr->name.rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } } } /* send it */ PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); } else { rc = PRTE_SUCCESS; } } } else if (PRTE_PROC_STATE_IOF_COMPLETE == state) { /* do NOT update the proc state as this can hit * while we are still trying to notify the HNP of * successful launch for short-lived procs */ PRTE_FLAG_SET(pdata, PRTE_PROC_FLAG_IOF_COMPLETE); /* Release the stdin IOF file descriptor for this child, if one * was defined. File descriptors for the other IOF channels - stdout, * stderr, and stddiag - were released when their associated pipes * were cleared and closed due to termination of the process * Do this after we handle termination in case the IOF needs * to check to see if all procs from the job are actually terminated */ if (NULL != prte_iof.close) { prte_iof.close(proc, PRTE_IOF_STDALL); } if (PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_WAITPID) && !PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_RECORDED)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } } else if (PRTE_PROC_STATE_WAITPID_FIRED == state) { /* do NOT update the proc state as this can hit * while we are still trying to notify the HNP of * successful launch for short-lived procs */ PRTE_FLAG_SET(pdata, PRTE_PROC_FLAG_WAITPID); if (PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_IOF_COMPLETE) && !PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_RECORDED)) { PRTE_ACTIVATE_PROC_STATE(proc, PRTE_PROC_STATE_TERMINATED); } } else if (PRTE_PROC_STATE_TERMINATED == state) { /* if this proc has not already recorded as terminated, then * update the accounting here */ if (!PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_RECORDED)) { jdata->num_terminated++; } /* update the proc state */ PRTE_FLAG_SET(pdata, PRTE_PROC_FLAG_RECORDED); PRTE_FLAG_UNSET(pdata, PRTE_PROC_FLAG_ALIVE); pdata->state = state; /* if we are trying to terminate and our routes are * gone, then terminate ourselves IF no local procs * remain (might be some from another job) */ if (prte_prteds_term_ordered && 0 == pmix_list_get_size(&prte_rml_base.children)) { for (i = 0; i < prte_local_children->size; i++) { pdata = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL != pdata && PRTE_FLAG_TEST(pdata, PRTE_PROC_FLAG_ALIVE)) { /* at least one is still alive */ PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted all routes gone but proc %s still alive", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&pdata->name))); goto cleanup; } } /* call our appropriate exit procedure */ PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted all routes and children gone - exiting", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_JOB_STATE_DAEMONS_TERMINATED); goto cleanup; } /* track job status */ if (jdata->num_terminated == jdata->num_local_procs && !prte_get_attribute(&jdata->attributes, PRTE_JOB_TERM_NOTIFIED, NULL, PMIX_BOOL)) { /* pack update state command */ cmd = PRTE_PLM_UPDATE_PROC_STATE; PMIX_DATA_BUFFER_CREATE(alert); rc = PMIx_Data_pack(NULL, alert, &cmd, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* pack the job info */ if (PRTE_SUCCESS != (rc = pack_state_update(alert, jdata))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); goto cleanup; } /* send it */ PMIX_OUTPUT_VERBOSE((5, prte_state_base_framework.framework_output, "%s state:prted: SENDING JOB LOCAL TERMINATION UPDATE FOR JOB %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); PRTE_RML_SEND(rc, PRTE_PROC_MY_HNP->rank, alert, PRTE_RML_TAG_PLM); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(alert); } /* mark that we sent it so we ensure we don't do it again */ prte_set_attribute(&jdata->attributes, PRTE_JOB_TERM_NOTIFIED, PRTE_ATTR_LOCAL, NULL, PMIX_BOOL); /* cleanup the procs as these are gone */ for (i = 0; i < prte_local_children->size; i++) { pptr = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i); if (NULL == pptr) { continue; } /* if this child is part of the job... */ if (PMIX_CHECK_NSPACE(pptr->name.nspace, jdata->nspace)) { /* clear the entry in the local children */ pmix_pointer_array_set_item(prte_local_children, i, NULL); PMIX_RELEASE(pptr); // maintain accounting } } /* tell the IOF that the job is complete */ if (NULL != prte_iof.complete) { prte_iof.complete(jdata); } /* tell the PMIx subsystem the job is complete */ PRTE_PMIX_CONSTRUCT_LOCK(&lock); PMIx_server_deregister_nspace(jdata->nspace, opcbfunc, &lock); PRTE_PMIX_WAIT_THREAD(&lock); PRTE_PMIX_DESTRUCT_LOCK(&lock); /* release the resources */ if (NULL != jdata->map) { map = jdata->map; for (index = 0; index < map->nodes->size; index++) { node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, index); if (NULL == node) { continue; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:prted releasing procs from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name)); for (i = 0; i < node->procs->size; i++) { pptr = (prte_proc_t *) pmix_pointer_array_get_item(node->procs, i); if (NULL == pptr) { continue; } if (!PMIX_CHECK_NSPACE(pptr->name.nspace, jdata->nspace)) { /* skip procs from another job */ continue; } app = (prte_app_context_t*) pmix_pointer_array_get_item(jdata->apps, pptr->app_idx); if (!PRTE_FLAG_TEST(app, PRTE_APP_FLAG_TOOL) && !PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_TOOL)) { node->slots_inuse--; node->num_procs--; } PMIX_OUTPUT_VERBOSE((2, prte_state_base_framework.framework_output, "%s state:prted releasing proc %s from node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&pptr->name), node->name)); /* set the entry in the node array to NULL */ pmix_pointer_array_set_item(node->procs, i, NULL); /* release the proc once for the map entry */ PMIX_RELEASE(pptr); } /* set the node location to NULL */ pmix_pointer_array_set_item(map->nodes, index, NULL); /* maintain accounting */ PMIX_RELEASE(node); /* flag that the node is no longer in a map */ PRTE_FLAG_UNSET(node, PRTE_NODE_FLAG_MAPPED); } PMIX_RELEASE(map); jdata->map = NULL; } /* if requested, check fd status for leaks */ if (prte_state_base.run_fdcheck) { prte_state_base_check_fds(jdata); } /* if ompi-server is around, then notify it to purge * any session-related info */ if (NULL != prte_data_server_uri) { PMIX_LOAD_PROCID(&target, jdata->nspace, PMIX_RANK_WILDCARD); prte_state_base_notify_data_server(&target); } /* cleanup the job info */ pmix_pointer_array_set_item(prte_job_data, jdata->index, NULL); PMIX_RELEASE(jdata); } } cleanup: PMIX_RELEASE(caddy); } static int pack_state_for_proc(pmix_data_buffer_t *alert, prte_proc_t *child) { int rc; /* pack the child's vpid */ rc = PMIx_Data_pack(NULL, alert, &child->name.rank, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack the pid */ rc = PMIx_Data_pack(NULL, alert, &child->pid, 1, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack its state */ rc = PMIx_Data_pack(NULL, alert, &child->state, 1, PMIX_UINT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } /* pack its exit code */ rc = PMIx_Data_pack(NULL, alert, &child->exit_code, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } return PRTE_SUCCESS; } static int pack_state_update(pmix_data_buffer_t *alert, prte_job_t *jdata) { int i, rc; prte_proc_t *child; pmix_rank_t null = PMIX_RANK_INVALID; /* pack the jobid */ rc = PMIx_Data_pack(NULL, alert, &jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } for (i = 0; i < prte_local_children->size; i++) { if (NULL == (child = (prte_proc_t *) pmix_pointer_array_get_item(prte_local_children, i))) { continue; } /* if this child is part of the job and has not been * previously reported... */ if (PMIX_CHECK_NSPACE(child->name.nspace, jdata->nspace) && !PRTE_FLAG_TEST(child, PRTE_PROC_FLAG_TERM_REPORTED)) { if (PRTE_SUCCESS != (rc = pack_state_for_proc(alert, child))) { PRTE_ERROR_LOG(rc); return rc; } PRTE_FLAG_SET(child, PRTE_PROC_FLAG_TERM_REPORTED); } } /* flag that this job is complete so the receiver can know */ rc = PMIx_Data_pack(NULL, alert, &null, 1, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/state/prted/Makefile.am0000664000175000017500000000234415203542132021463 0ustar alastairalastair# # Copyright (c) 2011 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ state_prted.h \ state_prted_component.c \ state_prted.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_state_prted_DSO component_noinst = component_install = prte_mca_state_prted.la else component_noinst = libprtemca_state_prted.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_state_prted_la_SOURCES = $(sources) prte_mca_state_prted_la_LDFLAGS = -module -avoid-version prte_mca_state_prted_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_state_prted_la_SOURCES =$(sources) libprtemca_state_prted_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/prtedl/0000775000175000017500000000000015203542132016460 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/prtedl/dlopen/0000775000175000017500000000000015203542132017741 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/prtedl/dlopen/prtedl_dlopen_module.c0000664000175000017500000001620015203542132024304 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved. * Copyright (c) 2018-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include #include #include #include #include #include #include "constants.h" #include "src/mca/prtedl/prtedl.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_printf.h" #include "prtedl_dlopen.h" /* * Trivial helper function to avoid replicating code */ static void do_dlopen(const char *fname, int flags, void **handle, char **err_msg) { assert(handle); *handle = dlopen(fname, flags); if (NULL != err_msg) { if (NULL != *handle) { *err_msg = NULL; } else { *err_msg = dlerror(); } } } static int dlopen_open(const char *fname, bool use_ext, bool private_namespace, prte_dl_handle_t **handle, char **err_msg) { assert(handle); *handle = NULL; /* Setup the dlopen flags */ int flags = RTLD_LAZY; if (private_namespace) { flags |= RTLD_LOCAL; } else { flags |= RTLD_GLOBAL; } /* If the caller wants to use filename extensions, loop through them */ void *local_handle = NULL; if (use_ext && NULL != fname) { int i, rc; char *ext; for (i = 0, ext = prte_mca_prtedl_dlopen_component.filename_suffixes[i]; NULL != ext; ext = prte_mca_prtedl_dlopen_component.filename_suffixes[++i]) { char *name; pmix_asprintf(&name, "%s%s", fname, ext); if (NULL == name) { return PRTE_ERR_IN_ERRNO; } /* Does the file exist? */ struct stat buf; if (stat(name, &buf) < 0) { if (NULL != err_msg) { rc = asprintf(err_msg, "File %s not found", name); if (0 > rc) { free(name); return PRTE_ERR_OUT_OF_RESOURCE; } } free(name); continue; } /* Yes, the file exists -- try to dlopen it. If we can't dlopen it, bail. */ do_dlopen(name, flags, &local_handle, err_msg); free(name); break; } } /* Otherwise, the caller does not want to use filename extensions, so just use the single filename that the caller provided */ else { do_dlopen(fname, flags, &local_handle, err_msg); } if (NULL != local_handle) { *handle = calloc(1, sizeof(prte_dl_handle_t)); (*handle)->dlopen_handle = local_handle; #if PRTE_ENABLE_DEBUG if (NULL != fname) { (*handle)->filename = strdup(fname); } else { (*handle)->filename = strdup("(null)"); } #endif } return (NULL != local_handle) ? PRTE_SUCCESS : PRTE_ERROR; } static int dlopen_lookup(prte_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg) { assert(handle); assert(handle->dlopen_handle); assert(symbol); assert(ptr); *ptr = dlsym(handle->dlopen_handle, symbol); if (NULL != *ptr) { return PRTE_SUCCESS; } if (NULL != err_msg) { *err_msg = dlerror(); } return PRTE_ERROR; } static int dlopen_close(prte_dl_handle_t *handle) { assert(handle); int ret; ret = dlclose(handle->dlopen_handle); #if PRTE_ENABLE_DEBUG free(handle->filename); #endif free(handle); return ret; } /* * Scan all the files in a directory (or path) and invoke a callback * on each one. */ static int dlopen_foreachfile(const char *search_path, int (*func)(const char *filename, void *data), void *data) { int ret; DIR *dp = NULL; char **dirs = NULL; char **good_files = NULL; dirs = PMIx_Argv_split(search_path, PRTE_ENV_SEP); for (int i = 0; NULL != dirs && NULL != dirs[i]; ++i) { dp = opendir(dirs[i]); if (NULL == dp) { ret = PRTE_ERR_IN_ERRNO; goto error; } struct dirent *de; while (NULL != (de = readdir(dp))) { /* Make the absolute path name */ char *abs_name = NULL; pmix_asprintf(&abs_name, "%s/%s", dirs[i], de->d_name); if (NULL == abs_name) { ret = PRTE_ERR_IN_ERRNO; goto error; } /* Stat the file */ struct stat buf; if (stat(abs_name, &buf) < 0) { free(abs_name); ret = PRTE_ERR_IN_ERRNO; goto error; } /* Skip if not a file */ if (!S_ISREG(buf.st_mode)) { free(abs_name); continue; } /* Find the suffix */ char *ptr = strrchr(abs_name, '.'); if (NULL != ptr) { /* Skip libtool files */ if (strcmp(ptr, ".la") == 0 || strcmp(ptr, ".lo") == 0) { free(abs_name); continue; } *ptr = '\0'; } /* Have we already found this file? Or already found a file with the same basename (but different suffix)? */ bool found = false; for (int j = 0; NULL != good_files && NULL != good_files[j]; ++j) { if (strcmp(good_files[j], abs_name) == 0) { found = true; break; } } if (!found) { PMIx_Argv_append_nosize(&good_files, abs_name); } free(abs_name); } closedir(dp); } dp = NULL; /* Invoke the callback on all the found files */ if (NULL != good_files) { for (int i = 0; NULL != good_files[i]; ++i) { ret = func(good_files[i], data); if (PRTE_SUCCESS != ret) { goto error; } } } ret = PRTE_SUCCESS; error: if (NULL != dp) { closedir(dp); } if (NULL != dirs) { PMIx_Argv_free(dirs); } if (NULL != good_files) { PMIx_Argv_free(good_files); } return ret; } /* * Module definition */ prte_prtedl_base_module_t prte_prtedl_dlopen_module = {.open = dlopen_open, .lookup = dlopen_lookup, .close = dlopen_close, .foreachfile = dlopen_foreachfile}; prrte-4.1.1~rc2/src/mca/prtedl/dlopen/configure.m40000664000175000017500000000464415203542132022174 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Amazon.com, Inc. or its affiliates. # All Rights reserved. # Copyright (c) 2023 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AC_DEFUN([MCA_prte_prtedl_dlopen_PRIORITY], [80]) # # Force this component to compile in static-only mode # AC_DEFUN([MCA_prte_prtedl_dlopen_COMPILE_MODE], [ AC_MSG_CHECKING([for MCA component $1:$2 compile mode]) $3="static" AC_MSG_RESULT([$$3]) ]) # MCA_prte_prtedl_pdlopen_POST_CONFIG() # --------------------------------- AC_DEFUN([MCA_prte_prtedl_pdlopen_POST_CONFIG],[ # If we won, then do all the rest of the setup AS_IF([test "$1" = "1"], [ # Add some stuff to CPPFLAGS so that the rest of the source # tree can be built LDFLAGS="$LDFLAGS $prte_prtedl_pdlopen_ADD_LDFLAGS" LIBS="$LIBS $prte_prtedl_pdlopen_ADD_LIBS" ]) ])dnl # MCA_prtedl_dlopen_CONFIG([action-if-can-compile], # [action-if-cant-compile]) # ------------------------------------------------ AC_DEFUN([MCA_prte_prtedl_dlopen_CONFIG],[ AC_CONFIG_FILES([src/mca/prtedl/dlopen/Makefile]) dnl This is effectively a back-door for PRTE developers to dnl force the use of the libltprtedl prtedl component. AC_ARG_ENABLE([prte-dlopen], [AS_HELP_STRING([--disable-prte-dlopen], [Disable the "dlopen" PRTE DL component (and probably force the use of the "libltdl" DL component). This option should really only be used by PRTE developers. You are probably actually looking for the "--disable-dlopen" option, which disables all dlopen-like functionality from PRTE.]) ]) prte_prtedl_dlopen_happy=no AS_IF([test "$enable_prte_dlopen" != "no"], [OAC_CHECK_PACKAGE([dlopen], [prte_prtedl_dlopen], [dlfcn.h], [dl], [dlopen], [prte_prtedl_dlopen_happy=yes], [prte_prtedl_dlopen_happy=no]) ]) AS_IF([test "$prte_prtedl_dlopen_happy" = "yes"], [prtedl_dlopen_ADD_LIBS=$prte_prtedl_dlopen_LIBS prtedl_dlopen_WRAPPER_EXTRA_LIBS=$prte_prtedl_dlopen_LIBS $1], [$2]) AC_SUBST(prte_prtedl_dlopen_LIBS) ]) prrte-4.1.1~rc2/src/mca/prtedl/dlopen/prtedl_dlopen_component.c0000664000175000017500000000744615203542132025035 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/prtedl/prtedl.h" #include "src/util/pmix_argv.h" #include "prtedl_dlopen.h" /* * Public string showing the sysinfo ompi_linux component version number */ const char *prte_mca_prtedl_dlopen_component_version_string = "PRTE prtedl dlopen MCA component version " PRTE_VERSION; /* * Local functions */ static int dlopen_component_register(void); static int dlopen_component_open(void); static int dlopen_component_close(void); static int dlopen_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_prtedl_dlopen_component_t prte_mca_prtedl_dlopen_component = { .base = { /* Fill in the mca_prtedl_base_component_t */ .base_version = { PRTE_DL_BASE_VERSION_1_0_0, /* Component name and version */ .pmix_mca_component_name = "dlopen", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component functions */ .pmix_mca_register_component_params = dlopen_component_register, .pmix_mca_open_component = dlopen_component_open, .pmix_mca_close_component = dlopen_component_close, .pmix_mca_query_component = dlopen_component_query, }, /* The prtedl framework members */ .priority = 80 }, }; PMIX_MCA_BASE_COMPONENT_INIT(prte, prtedl, dlopen) static int dlopen_component_register(void) { int ret; prte_mca_prtedl_dlopen_component.filename_suffixes_mca_storage = ".so,.dylib,.dll,.sl"; ret = pmix_mca_base_component_var_register(&prte_mca_prtedl_dlopen_component.base.base_version, "filename_suffixes", "Comma-delimited list of filename suffixes that the PRTE dlopen component will try", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_prtedl_dlopen_component.filename_suffixes_mca_storage); if (ret < 0) { return ret; } prte_mca_prtedl_dlopen_component.filename_suffixes = PMIx_Argv_split(prte_mca_prtedl_dlopen_component.filename_suffixes_mca_storage, ','); return PRTE_SUCCESS; } static int dlopen_component_open(void) { return PRTE_SUCCESS; } static int dlopen_component_close(void) { if (NULL != prte_mca_prtedl_dlopen_component.filename_suffixes) { PMIx_Argv_free(prte_mca_prtedl_dlopen_component.filename_suffixes); prte_mca_prtedl_dlopen_component.filename_suffixes = NULL; } return PRTE_SUCCESS; } static int dlopen_component_query(pmix_mca_base_module_t **module, int *priority) { /* The priority value is somewhat meaningless here; by src/mca/prtedl/configure.m4, there's at most one component available. */ *priority = prte_mca_prtedl_dlopen_component.base.priority; *module = &prte_prtedl_dlopen_module.super; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/prtedl/dlopen/prtedl_dlopen.h0000664000175000017500000000214015203542132022742 0ustar alastairalastair/* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_DL_DLOPEN #define PRTE_DL_DLOPEN #include "prte_config.h" #include "src/mca/prtedl/prtedl.h" PRTE_EXPORT extern prte_prtedl_base_module_t prte_prtedl_dlopen_module; /* * Dynamic library handles generated by this component. * * If we're debugging, keep a copy of the name of the file we've opened. */ struct prte_dl_handle_t { void *dlopen_handle; #if PRTE_ENABLE_DEBUG void *filename; #endif }; typedef struct { prte_prtedl_base_component_t base; char *filename_suffixes_mca_storage; char **filename_suffixes; } prte_prtedl_dlopen_component_t; PRTE_EXPORT extern prte_prtedl_dlopen_component_t prte_mca_prtedl_dlopen_component; #endif /* PRTE_DL_DLOPEN */ prrte-4.1.1~rc2/src/mca/prtedl/dlopen/Makefile.am0000664000175000017500000000140615203542132021776 0ustar alastairalastair# # Copyright (c) 2004-2010 The Trustees of Indiana University. # All rights reserved. # Copyright (c) 2014-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ prtedl_dlopen.h \ prtedl_dlopen_component.c \ prtedl_dlopen_module.c # This component will only ever be built statically -- never as a DSO. noinst_LTLIBRARIES = libprtemca_prtedl_dlopen.la libprtemca_prtedl_dlopen_la_SOURCES = $(sources) libprtemca_prtedl_dlopen_la_LDFLAGS = -module -avoid-version libprtemca_prtedl_dlopen_la_LIBADD = $(prte_prtedl_dlopen_LIBS) prrte-4.1.1~rc2/src/mca/prtedl/prtedl.h0000664000175000017500000001453115203542132020127 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * @file * * Dynamic library framework * * General Description: * * This framework provides portable access to dlopen- and prtedlsym-like * functionality, very similar to Libtool's libltprtedl. Indeed, one of * the components in this framework will use libltprtedl, if it is * present/available. However, on some common types systems where * libltprtedl headers and libraries are *not* available, we can support * plugins via this simple framework. * * This is a compile-time framework: a single component will be * selected by the priority that its configure.m4 provides. All other * components will be ignored (i.e., not built/not part of the * installation). Meaning: the static_components of the prtedl framework * will always contain 0 or 1 components. * * SIDENOTE: PRTE used to embed libltprtedl. However, as of early * 2015, this became problematic, for a variety of complex and * uninteresting reasons (see the following if you care about the * details: https://github.com/open-mpi/ompi/issues/311, * http://debbugs.gnu.org/cgi/bugreport.cgi?bug=19370, * https://github.com/open-mpi/ompi/pull/366, * https://github.com/open-mpi/ompi/pull/390). That being said, we, * as a developer community, still wanted to be able to natively use * DSOs by default. A small/simple framework for DL functionality, * along with a simple component that supports dlopen/prtedlsym on POSIX * platforms and another component that natively uses libltprtedl seemed * like a good solution. */ #ifndef PRTE_MCA_DL_DL_H #define PRTE_MCA_DL_DL_H #include "prte_config.h" #include "src/pmix/pmix-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" BEGIN_C_DECLS /** * Handle for an opened file */ struct prte_dl_handle_t; typedef struct prte_dl_handle_t prte_dl_handle_t; /** * Dynamically open the file specified. * * Arguments: * fname = Base filename to open. If NULL, open this process. * use_ext = If true, try various filename suffixes that are * relevant on this platform (e.g., .so, .prtedll, .dylib). If * false, just use exactly whatever was passed as fname. * private = If true, open the file in a private namespace. * Otherwise, open the file in a global namespace. * handle = Upon successful open, a handle to the opened file will * be returned. * err_msg= if non-NULL and !=PRTE_SUCCESS is returned, will point to a * string error message * * Returns: * PRTE_SUCCESS on success, or PRTE_ERROR * * Space for the handle must be allocated by the module (it can be * freed during the call to prte_prtedl_base_module_prtedlclose_fn_t). * * The err_msg points to an internal string and should not be altered * or freed by the caller. The contents of the err_msg string may * change after successive calls to prte_prtedl API calls. */ typedef int (*prte_prtedl_base_module_open_fn_t)(const char *fname, bool use_ext, bool private_namespace, prte_dl_handle_t **handle, char **err_msg); /** * Lookup a symbol in an opened file. * * Arguments: * handle = handle of a previously dynamically opened file * symbol = name of the symbol to lookup * ptr = if found, a pointer to the symbol. Otherwise, NULL. * err_msg= if non-NULL and !=PRTE_SUCCESS is returned, will point to a * string error message * Returns: * PRTE_SUCCESS on success, or PRTE_ERROR * * * The err_msg points to an internal string and should not be altered * or freed by the caller. The contents of the err_msg string may * change after successive calls to prte_prtedl API calls. */ typedef int (*prte_prtedl_base_module_lookup_fn_t)(prte_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg); /** * Dynamically close a previously dynamically-opened file. * * Arguments: * handle = handle of a previously dynamically opened file. * Returns: * PRTE_SUCCESS on success, or PRTE_ERROR * * This function should close the file and free and resources * associated with it (e.g., whatever is cached on the handle). */ typedef int (*prte_prtedl_base_module_close_fn_t)(prte_dl_handle_t *handle); /** * Search through a path of directories, invoking a callback on each * unique regular (non-Libtool) file basename found (e.g., will only * be invoked once for the files "foo.la" and "foo.so", with the * parameter "foo"). * * Arguments: * path = PRTE_ENV_SEP-delimited list of directories * cb_func= function to invoke on each filename found * data = context for callback function * Returns: * PRTE_SUCESS on success, PRTE_ERR* otherwise */ typedef int (*prte_prtedl_base_module_foreachfile_fn_t)( const char *search_path, int (*cb_func)(const char *filename, void *context), void *context); /** * Structure for DL components. */ struct prte_prtedl_base_component_1_0_0_t { /** MCA base component */ pmix_mca_base_component_t base_version; /** Default priority */ int priority; }; typedef struct prte_prtedl_base_component_1_0_0_t prte_prtedl_base_component_1_0_0_t; typedef struct prte_prtedl_base_component_1_0_0_t prte_prtedl_base_component_t; /** * Structure for DL modules */ struct prte_prtedl_base_module_1_0_0_t { pmix_mca_base_module_t super; /** Open / close */ prte_prtedl_base_module_open_fn_t open; prte_prtedl_base_module_close_fn_t close; /** Lookup a symbol */ prte_prtedl_base_module_lookup_fn_t lookup; /** Iterate looking for files */ prte_prtedl_base_module_foreachfile_fn_t foreachfile; }; typedef struct prte_prtedl_base_module_1_0_0_t prte_prtedl_base_module_1_0_0_t; typedef struct prte_prtedl_base_module_1_0_0_t prte_prtedl_base_module_t; /** * Macro for use in components that are of type DL */ #define PRTE_DL_BASE_VERSION_1_0_0 PRTE_MCA_BASE_VERSION_3_0_0("prtedl", 1, 0, 0) END_C_DECLS #endif /* PRTE_MCA_DL_DL_H */ prrte-4.1.1~rc2/src/mca/prtedl/configure.m40000664000175000017500000000664715203542132020720 0ustar alastairalastairdnl -*- shell-script -*- dnl dnl Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved dnl Copyright (c) 2019-2020 Intel, Inc. All rights reserved. dnl Copyright (c) 2023 Nanook Consulting. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow dnl dnl $HEADER$ dnl dnl There will only be one component used in this framework, and it will dnl be selected at configure time by priority. Components must set dnl their priorities in their configure.m4 file. dnl We only want one winning component (vs. STOP_AT_FIRST_PRIORITY, dnl which will allow all components of the same priority who succeed to dnl win) m4_define(MCA_prte_prtedl_CONFIGURE_MODE, STOP_AT_FIRST) AC_DEFUN([MCA_prte_prtedl_CONFIG],[ PRTE_HAVE_DL_SUPPORT=0 # If --disable-dlopen was used, then have all the components fail # (we still need to configure them all so that things like "make # dist" work", but we just want the MCA system to (artificially) # conclude that it can't build any of the components. AS_IF([test $PRTE_ENABLE_DLOPEN_SUPPORT -eq 0], [want_prtedl=0], [want_prtedl=1]) MCA_CONFIGURE_FRAMEWORK([prtedl], [$want_prtedl]) # If we found no suitable static prtedl component and dlopen support # was not specifically disabled, this is an error. AS_IF([test "$MCA_prte_prtedl_STATIC_COMPONENTS" = "" && \ test "$enable_dlopen" != "no"], [AC_MSG_WARN([Did not find a suitable static prte prtedl component]) AC_MSG_WARN([You might need to install libltld (and its headers) or]) AC_MSG_WARN([specify --disable-dlopen to configure.]) AC_MSG_ERROR([Cannot continue])]) # If we have a winning component (which, per above, will only # happen if --disable-dlopen was *not* specified), do some more # logic. AS_IF([test "$MCA_prte_prtedl_STATIC_COMPONENTS" != ""], [ # We had a winner -- w00t! PRTE_HAVE_DL_SUPPORT=1 # If we added any -L flags to ADD_LDFLAGS, then we (might) # need to add those directories to LD_LIBRARY_PATH. # Otherwise, if we try to AC RUN_IFELSE anything here in # configure, it might die because it can't find the libraries # we just linked against. PRTE_VAR_SCOPE_PUSH([prte_prtedl_base_found_l prte_prtedl_base_token prte_prtedl_base_tmp prte_prtedl_base_dir]) prte_prtedl_base_found_l=0 eval "prte_prtedl_base_tmp=\$prte_prtedl_${prte_prtedl_winner}_ADD_LIBS" for prte_prtedl_base_token in $prte_prtedl_base_tmp; do case $prte_prtedl_base_token in -l*) prte_prtedl_base_found_l=1 ;; esac done AS_IF([test $prte_prtedl_base_found_l -eq 1], [eval "prte_prtedl_base_tmp=\$prte_prtedl_${prte_prtedl_winner}_ADD_LDFLAGS" for prte_prtedl_base_token in $prte_prtedl_base_tmp; do case $prte_prtedl_base_token in -L*) prte_prtedl_base_dir=`echo $prte_prtedl_base_token | cut -c3-` export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$prte_prtedl_base_dir AC_MSG_WARN([Adding to LD_LIBRARY_PATH: $prte_prtedl_base_dir]) ;; esac done]) PRTE_VAR_SCOPE_POP ]) AC_DEFINE_UNQUOTED([PRTE_HAVE_DL_SUPPORT], [$PRTE_HAVE_DL_SUPPORT], [Whether the PRTE DL framework is functional or not]) ]) prrte-4.1.1~rc2/src/mca/prtedl/base/0000775000175000017500000000000015203542132017372 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/prtedl/base/prtedl_base_close.c0000664000175000017500000000132015203542132023203 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/prtedl/base/base.h" #include "src/mca/prtedl/prtedl.h" int prte_dl_base_close(void) { /* Close all available modules that are open */ return pmix_mca_base_framework_components_close(&prte_prtedl_base_framework, NULL); } prrte-4.1.1~rc2/src/mca/prtedl/base/prtedl_base_fns.c0000664000175000017500000000342215203542132022671 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** * This file is a simple set of wrappers around the selected PRTE DL * component (it's a compile-time framework with, at most, a single * component; see prtedl.h for details). */ #include "prte_config.h" #include "src/include/constants.h" #include "src/mca/prtedl/base/base.h" int prte_dl_open(const char *fname, bool use_ext, bool private_namespace, prte_dl_handle_t **handle, char **err_msg) { *handle = NULL; if (NULL != prte_prtedl && NULL != prte_prtedl->open) { return prte_prtedl->open(fname, use_ext, private_namespace, handle, err_msg); } return PRTE_ERR_NOT_SUPPORTED; } int prte_dl_lookup(prte_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg) { if (NULL != prte_prtedl && NULL != prte_prtedl->lookup) { return prte_prtedl->lookup(handle, symbol, ptr, err_msg); } return PRTE_ERR_NOT_SUPPORTED; } int prte_dl_close(prte_dl_handle_t *handle) { if (NULL != prte_prtedl && NULL != prte_prtedl->close) { return prte_prtedl->close(handle); } return PRTE_ERR_NOT_SUPPORTED; } int prte_dl_foreachfile(const char *search_path, int (*cb_func)(const char *filename, void *context), void *context) { if (NULL != prte_prtedl && NULL != prte_prtedl->foreachfile) { return prte_prtedl->foreachfile(search_path, cb_func, context); } return PRTE_ERR_NOT_SUPPORTED; } prrte-4.1.1~rc2/src/mca/prtedl/base/prtedl_base_select.c0000664000175000017500000000324515203542132023365 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2010 The Trustees of Indiana University. * All rights reserved. * * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #ifdef HAVE_UNISTD_H # include "unistd.h" #endif #include "src/include/constants.h" #include "src/mca/base/pmix_base.h" #include "src/mca/mca.h" #include "src/mca/prtedl/base/base.h" #include "src/mca/prtedl/prtedl.h" #include "src/util/pmix_output.h" int prte_dl_base_select(void) { int exit_status = PRTE_SUCCESS; prte_prtedl_base_component_t *best_component = NULL; prte_prtedl_base_module_t *best_module = NULL; /* * Select the best component */ if (PRTE_SUCCESS != pmix_mca_base_select("prtedl", prte_prtedl_base_framework.framework_output, &prte_prtedl_base_framework.framework_components, (pmix_mca_base_module_t **) &best_module, (pmix_mca_base_component_t **) &best_component, NULL)) { /* This will only happen if no component was selected */ exit_status = PRTE_ERROR; goto cleanup; } /* Save the winner */ prte_prtedl_base_selected_component = best_component; prte_prtedl = best_module; cleanup: return exit_status; } prrte-4.1.1~rc2/src/mca/prtedl/base/base.h0000664000175000017500000000540015203542132020454 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2017-2020 Intel, Inc. All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_DL_BASE_H #define PRTE_DL_BASE_H #include "prte_config.h" #include "src/mca/prtedl/prtedl.h" #include "src/util/pmix_environ.h" #include "src/mca/base/pmix_base.h" #include "src/mca/base/pmix_mca_base_framework.h" BEGIN_C_DECLS /** * Globals */ PRTE_EXPORT extern pmix_mca_base_framework_t prte_prtedl_base_framework; PRTE_EXPORT extern prte_prtedl_base_component_t *prte_prtedl_base_selected_component; PRTE_EXPORT extern prte_prtedl_base_module_t *prte_prtedl; /** * Initialize the DL MCA framework * * @retval PRTE_SUCCESS Upon success * @retval PRTE_ERROR Upon failures * * This function is invoked during prte_init(); */ PRTE_EXPORT int prte_dl_base_open(pmix_mca_base_open_flag_t flags); /** * Select an available component. * * @retval PRTE_SUCCESS Upon Success * @retval PRTE_NOT_FOUND If no component can be selected * @retval PRTE_ERROR Upon other failure * */ PRTE_EXPORT int prte_dl_base_select(void); /** * Finalize the DL MCA framework * * @retval PRTE_SUCCESS Upon success * @retval PRTE_ERROR Upon failures * * This function is invoked during prte_finalize(); */ PRTE_EXPORT int prte_dl_base_close(void); /** * Open a DSO * * (see prte_prtedl_base_module_open_ft_t in src/mca/prtedl/prtedl.h for * documentation of this function) */ PRTE_EXPORT int prte_dl_open(const char *fname, bool use_ext, bool private_namespace, prte_dl_handle_t **handle, char **err_msg); /** * Lookup a symbol in a DSO * * (see prte_prtedl_base_module_lookup_ft_t in src/mca/prtedl/prtedl.h for * documentation of this function) */ PRTE_EXPORT int prte_dl_lookup(prte_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg); /** * Close a DSO * * (see prte_prtedl_base_module_close_ft_t in src/mca/prtedl/prtedl.h for * documentation of this function) */ PRTE_EXPORT int prte_dl_close(prte_dl_handle_t *handle); /** * Iterate over files in a path * * (see prte_prtedl_base_module_foreachfile_ft_t in src/mca/prtedl/prtedl.h for * documentation of this function) */ PRTE_EXPORT int prte_dl_foreachfile(const char *search_path, int (*cb_func)(const char *filename, void *context), void *context); END_C_DECLS #endif /* PRTE_DL_BASE_H */ prrte-4.1.1~rc2/src/mca/prtedl/base/Makefile.am0000664000175000017500000000071715203542132021433 0ustar alastairalastair# # Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # headers += \ base/base.h libprtemca_prtedl_la_SOURCES += \ base/prtedl_base_open.c \ base/prtedl_base_close.c \ base/prtedl_base_select.c \ base/prtedl_base_fns.c prrte-4.1.1~rc2/src/mca/prtedl/base/prtedl_base_open.c0000664000175000017500000000414615203542132023050 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "src/mca/prtedl/base/base.h" #include "src/mca/prtedl/base/static-components.h" /* * Globals */ prte_prtedl_base_module_t *prte_prtedl = NULL; prte_prtedl_base_component_t *prte_prtedl_base_selected_component = NULL; /* * Function for finding and opening either all MCA components, * or the one that was specifically requested via a MCA parameter. * * Note that we really don't need this function -- we could specify a * NULL pointer in the framework declare and the base would do this * exact same thing. However, we need to have at least some * executable code in this file, or some linkers (cough cough OS X * cough cough) may not actually link in this .o file. */ int prte_dl_base_open(pmix_mca_base_open_flag_t flags) { /* Open up all available components */ return pmix_mca_base_framework_components_open(&prte_prtedl_base_framework, flags); } /* VERY IMPORTANT: This framework is static, and is opened before any other dyanmic frameworks are opened (which makes sense, of course). But we must mark this framework is NO_DSO so that the MCA framework base doesn't try to open any dynamic components in this framework. */ PMIX_MCA_BASE_FRAMEWORK_DECLARE(prte, prtedl, "Dynamic loader framework", NULL /* register */, prte_dl_base_open /* open */, NULL /* close */, prte_prtedl_base_static_components, PMIX_MCA_BASE_FRAMEWORK_FLAG_NO_DSO); prrte-4.1.1~rc2/src/mca/prtedl/Makefile.am0000664000175000017500000000211615203542132020514 0ustar alastairalastair# # Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # main library setup noinst_LTLIBRARIES = libprtemca_prtedl.la libprtemca_prtedl_la_SOURCES = # local files headers = prtedl.h libprtemca_prtedl_la_SOURCES += $(headers) # Ensure that the man pages are rebuilt if the prte_config.h file # changes; a "good enough" way to know if configure was run again (and # therefore the release date or version may have changed) $(nodist_man_MANS): $(top_builddir)/src/include/prte_config.h # Conditionally install the header files prtedir = $(prteincludedir)/$(subdir) nobase_prte_HEADERS = $(headers) include base/Makefile.am distclean-local: rm -f base/static-components.h rm -f $(nodist_man_MANS) prrte-4.1.1~rc2/src/mca/prtedl/libltdl/0000775000175000017500000000000015203542132020106 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/prtedl/libltdl/prtedl_libltdl_component.c0000664000175000017500000001132115203542132025332 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2015 Los Alamos National Security, Inc. All rights * reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/mca_base_var.h" #include "src/mca/prtedl/prtedl.h" #include "src/util/pmix_argv.h" #include "prtedl_libltdl.h" /* * Public string showing the sysinfo ompi_linux component version number */ const char *prte_mca_prtedl_libltdl_component_version_string = "PRTE prtedl libltdl MCA component version " PRTE_VERSION; /* * Local functions */ static int libltdl_component_register(void); static int libltdl_component_open(void); static int libltdl_component_close(void); static int libltdl_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_prtedl_libltdl_component_t prte_mca_prtedl_libltdl_component = { /* Fill in the mca_prtedl_base_component_t */ .base = { /* First, the mca_component_t struct containing meta information about the component itself */ .base_version = { PRTE_DL_BASE_VERSION_1_0_0, /* Component name and version */ .pmix_mca_component_name = "libltdl", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component functions */ .pmix_mca_register_component_params = libltdl_component_register, .pmix_mca_open_component = libltdl_component_open, .pmix_mca_close_component = libltdl_component_close, .pmix_mca_query_component = libltdl_component_query, }, /* The prtedl framework members */ .priority = 50 } /* Now fill in the libltdl component-specific members */ }; PMIX_MCA_BASE_COMPONENT_INIT(prte, prtedl, libltdl) static bool supported = PRTE_INT_TO_BOOL(PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE); static int libltdl_component_register(void) { /* Register an info param indicating whether we have lt_dladvise support or not */ pmix_mca_base_component_var_register(&prte_mca_prtedl_libltdl_component.base.base_version, "have_lt_dladvise", "Whether the version of libltdl that this component is " "built against supports lt_dladvise functionality or not", PMIX_MCA_BASE_VAR_TYPE_BOOL, &supported); return PRTE_SUCCESS; } static int libltdl_component_open(void) { if (lt_prtedlinit()) { return PRTE_ERROR; } #if PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE prte_mca_prtedl_libltdl_component_t *c = &prte_mca_prtedl_libltdl_component; if (lt_dladvise_init(&c->advise_private_noext)) { return PRTE_ERR_OUT_OF_RESOURCE; } if (lt_dladvise_init(&c->advise_private_ext) || lt_dladvise_ext(&c->advise_private_ext)) { return PRTE_ERR_OUT_OF_RESOURCE; } if (lt_dladvise_init(&c->advise_public_noext) || lt_dladvise_global(&c->advise_public_noext)) { return PRTE_ERR_OUT_OF_RESOURCE; } if (lt_dladvise_init(&c->advise_public_ext) || lt_dladvise_global(&c->advise_public_ext) || lt_dladvise_ext(&c->advise_public_ext)) { return PRTE_ERR_OUT_OF_RESOURCE; } #endif return PRTE_SUCCESS; } static int libltdl_component_close(void) { #if PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE prte_mca_prtedl_libltdl_component_t *c = &prte_mca_prtedl_libltdl_component; lt_dladvise_destroy(&c->advise_private_noext); lt_dladvise_destroy(&c->advise_private_ext); lt_dladvise_destroy(&c->advise_public_noext); lt_dladvise_destroy(&c->advise_public_ext); #endif lt_prtedlexit(); return PRTE_SUCCESS; } static int libltdl_component_query(pmix_mca_base_module_t **module, int *priority) { /* The priority value is somewhat meaningless here; by src/mca/prtedl/configure.m4, there's at most one component available. */ *priority = prte_mca_prtedl_libltdl_component.base.priority; *module = &prte_prtedl_libltdl_module.super; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/prtedl/libltdl/prtedl_libltdl_module.c0000664000175000017500000000654115203542132024625 0ustar alastairalastair/* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #include "prte_config.h" #include "constants.h" #include "src/mca/prtedl/prtedl.h" #include "prtedl_libltdl.h" static int libltdl_open(const char *fname, bool use_ext, bool private_namespace, prte_dl_handle_t **handle, char **err_msg) { assert(handle); *handle = NULL; if (NULL != err_msg) { *err_msg = NULL; } lt_prtedlhandle local_handle; #if PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE prte_prtedl_libltdl_component_t *c = &prte_prtedl_libltdl_component; if (use_ext && private_namespace) { local_handle = lt_dlopenadvise(fname, c->advise_private_ext); } else if (use_ext && !private_namespace) { local_handle = lt_dlopenadvise(fname, c->advise_public_ext); } else if (!use_ext && private_namespace) { local_handle = lt_dlopenadvise(fname, c->advise_private_noext); } else if (!use_ext && !private_namespace) { local_handle = lt_dlopenadvise(fname, c->advise_public_noext); } #else if (use_ext) { local_handle = lt_dlopenext(fname); } else { local_handle = lt_dlopen(fname); } #endif if (NULL != local_handle) { *handle = calloc(1, sizeof(prte_dl_handle_t)); (*handle)->ltdl_handle = local_handle; #if PRTE_ENABLE_DEBUG if (NULL != fname) { (*handle)->filename = strdup(fname); } else { (*handle)->filename = strdup("(null)"); } #endif return PRTE_SUCCESS; } if (NULL != err_msg) { *err_msg = strdup((char *) lt_dlerror()); } return PRTE_ERROR; } static int libltdl_lookup(prte_dl_handle_t *handle, const char *symbol, void **ptr, char **err_msg) { assert(handle); assert(handle->ltdl_handle); assert(symbol); assert(ptr); if (NULL != err_msg) { *err_msg = NULL; } *ptr = lt_dlsym(handle->ltdl_handle, symbol); if (NULL != *ptr) { return PRTE_SUCCESS; } if (NULL != err_msg) { *err_msg = (char *) lt_dlerror(); } return PRTE_ERROR; } static int libltdl_close(prte_dl_handle_t *handle) { assert(handle); int ret; ret = lt_dlclose(handle->ltdl_handle); #if PRTE_ENABLE_DEBUG free(handle->filename); #endif free(handle); return ret; } static int libltdl_foreachfile(const char *search_path, int (*func)(const char *filename, void *data), void *data) { assert(search_path); assert(func); int ret = lt_dlforeachfile(search_path, func, data); return (0 == ret) ? PRTE_SUCCESS : PRTE_ERROR; } /* * Module definition */ prte_prtedl_base_module_t prte_prtedl_libltdl_module = {.open = libltdl_open, .lookup = libltdl_lookup, .close = libltdl_close, .foreachfile = libltdl_foreachfile}; prrte-4.1.1~rc2/src/mca/prtedl/libltdl/prtedl_libltdl.h0000664000175000017500000000263015203542132023260 0ustar alastairalastair/* * Copyright (c) 2015-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_DL_LIBLTDL #define PRTE_DL_LIBLTDL #include "prte_config.h" #include "src/mca/prtedl/prtedl.h" #include PRTE_EXPORT extern prte_prtedl_base_module_t prte_prtedl_libltdl_module; /* * Dynamic library handles generated by this component. * * If we're debugging, keep a copy of the name of the file we've opened. */ struct prte_dl_handle_t { lt_prtedlhandle ltprtedl_handle; #if PRTE_ENABLE_DEBUG char *filename; #endif }; typedef struct { prte_prtedl_base_component_t base; #if PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE /* If the version of libltdl that we are compiling against has lt_dladvise, use it to support opening DSOs in a variety of modes. */ lt_dladvise advise_private_noext; lt_dladvise advise_private_ext; lt_dladvise advise_public_noext; lt_dladvise advise_public_ext; #endif } prte_prtedl_libltdl_component_t; PRTE_EXPORT extern prte_prtedl_libltdl_component_t prte_mca_prtedl_libltdl_component; #endif /* PRTE_DL_LIBLTDL */ prrte-4.1.1~rc2/src/mca/prtedl/libltdl/configure.m40000664000175000017500000000712715203542132022340 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2021-2023 Nanook Consulting. All rights reserved. # Copyright (c) 2022 Amazon.com, Inc. or its affiliates. # All Rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AC_DEFUN([MCA_prte_prtedl_libltdl_PRIORITY], [50]) # # Force this component to compile in static-only mode # AC_DEFUN([MCA_prte_prtedl_libltdl_COMPILE_MODE], [ AC_MSG_CHECKING([for MCA component $1:$2 compile mode]) $3="static" AC_MSG_RESULT([$$3]) ]) # MCA_prte_prtedl_libltdl_POST_CONFIG() # --------------------------------- AC_DEFUN([MCA_prte_prtedl_libltdl_POST_CONFIG],[ # If we won, then do all the rest of the setup AS_IF([test "$1" = "1"], [ # Add some stuff to CPPFLAGS so that the rest of the source # tree can be built LDFLAGS="$LDFLAGS $prte_prtedl_libltdl_ADD_LDFLAGS" LIBS="$LIBS $prte_prtedl_libltdl_ADD_LIBS" ]) ])dnl # MCA_prtedl_libltdl_CONFIG([action-if-can-compile], # [action-if-cant-compile]) # ------------------------------------------------ AC_DEFUN([MCA_prte_prtedl_libltdl_CONFIG],[ PRTE_VAR_SCOPE_PUSH([CPPFLAGS_save LDFLAGS_save LIBS_save]) AC_CONFIG_FILES([src/mca/prtedl/libltdl/Makefile]) # Add --with options AC_ARG_WITH([libltdl], [AS_HELP_STRING([--with-libltdl(=DIR)], [Build libltdl support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])]) AC_ARG_WITH([libltdl-libdir], [AS_HELP_STRING([--with-libltdl-libdir=DIR], [Search for libltdl libraries in DIR])]) prte_prtedl_libltdl_happy=no AS_IF([test "$with_libltdl" != "no"], [OAC_CHECK_PACKAGE([libltdl], [prte_prtedl_libltdl], [ltprtedl.h], [ltprtedl], [lt_dlopen], [prte_prtedl_libltdl_happy=yes], [prte_prtedl_libltdl_happy=no]) ]) # If we have libltdl, do we have lt_dladvise? prte_prtedl_libltdl_have_lt_dladvise=0 AS_IF([test "$prte_prtedl_libltdl_happy" = "yes"], [CPPFLAGS_save=$CPPFLAGS LDFLAGS_save=$LDFLAGS LIBS_save=$LIBS CPPFLAGS="$prte_prtedl_libltdl_CPPFLAGS $CPPFLAGS" LDFLAGS="$prte_prtedl_libltdl_LDFLAGS $LDFLAGS" LIBS="$prte_prtedl_libltdl_LIBS $LIBS" AC_CHECK_FUNC([lt_dladvise_init], [prte_prtedl_libltdl_have_lt_dladvise=1]) CPPFLAGS=$CPPFLAGS_save LDFLAGS=$LDFLAGS_save LIBS=$LIBS_save ]) AC_DEFINE_UNQUOTED(PRTE_DL_LIBLTDL_HAVE_LT_DLADVISE, [$prte_prtedl_libltdl_have_lt_dladvise], [Whether we have lt_dladvise or not]) AS_IF([test "$prte_prtedl_libltdl_happy" = "yes"], [prte_prtedl_libltdl_ADD_CPPFLAGS=$prte_prtedl_libltdl_CPPFLAGS prte_prtedl_libltdl_ADD_LDFLAGS=$prte_prtedl_libltdl_LDFLAGS prte_prtedl_libltdl_ADD_LIBS=$prte_prtedl_libltdl_LIBS $1], [AS_IF([test ! -z "$with_libltdl" && \ test "$with_libltdl" != "no"], [AC_MSG_WARN([Libltdl support requested (via --with-libltdl) but not found.]) AC_MSG_ERROR([Cannot continue.])]) $2]) AC_SUBST(prte_prtedl_libltdl_CPPFLAGS) AC_SUBST(prte_prtedl_libltdl_LDFLAGS) AC_SUBST(prte_prtedl_libltdl_LIBS) PRTE_VAR_SCOPE_POP ]) prrte-4.1.1~rc2/src/mca/prtedl/libltdl/Makefile.am0000664000175000017500000000161115203542132022141 0ustar alastairalastair# # Copyright (c) 2004-2010 The Trustees of Indiana University. # All rights reserved. # Copyright (c) 2014-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2019-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # sources = \ prtedl_libltdl.h \ prtedl_libltdl_component.c \ prtedl_libltdl_module.c # This component will only ever be built statically -- never as a DSO. noinst_LTLIBRARIES = libprtemca_prtedl_libltdl.la libprtemca_prtedl_libltdl_la_SOURCES = $(sources) libprtemca_prtedl_libltdl_la_CPPFLAGS = $(prte_prtedl_libltdl_CPPFLAGS) libprtemca_prtedl_libltdl_la_LDFLAGS = \ $(prte_prtedl_libltdl_LDFLAGS) \ -module -avoid-version libprtemca_prtedl_libltdl_la_LIBADD = $(prte_prtedl_libltdl_LIBS) prrte-4.1.1~rc2/src/mca/plm/0000775000175000017500000000000015203542132015756 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/plm/plm.h0000664000175000017500000001011415203542132016714 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ /** @file: * * The Process Lifecycle Management (PLM) subsystem serves as the central * switchyard for all process management activities, including * resource allocation, process mapping, process launch, and process * monitoring. */ #ifndef PRTE_PLM_H #define PRTE_PLM_H /* * includes */ #include "prte_config.h" #include "types.h" #include "src/class/pmix_pointer_array.h" #include "src/mca/mca.h" #include "src/pmix/pmix-internal.h" #include "src/runtime/prte_globals.h" #include "plm_types.h" BEGIN_C_DECLS /* * Component functions - all MUST be provided */ /* * allow the selected module to initialize */ typedef int (*prte_plm_base_module_init_fn_t)(void); /* * Spawn a job - this is a non-blocking function! */ typedef int (*prte_plm_base_module_spawn_fn_t)(prte_job_t *jdata); /* * Remote spawn - spawn called by a daemon to launch a process on its own */ typedef int (*prte_plm_base_module_remote_spawn_fn_t)(void); /* * Entry point to set the HNP name */ typedef int (*prte_plm_base_module_set_hnp_name_fn_t)(void); /** * Cleanup resources held by module. */ typedef int (*prte_plm_base_module_finalize_fn_t)(void); /** * Terminate any processes launched for the respective jobid by * this component. */ typedef int (*prte_plm_base_module_terminate_job_fn_t)(pmix_nspace_t); /** * Terminate the daemons */ typedef int (*prte_plm_base_module_terminate_orteds_fn_t)(void); /** * Terminate an array of specific procs */ typedef int (*prte_plm_base_module_terminate_procs_fn_t)(pmix_pointer_array_t *procs); /** * Signal any processes launched for the respective jobid by * this component. */ typedef int (*prte_plm_base_module_signal_job_fn_t)(pmix_nspace_t, int32_t); /** * plm module version 1.0.0 */ struct prte_plm_base_module_1_0_0_t { prte_plm_base_module_init_fn_t init; prte_plm_base_module_set_hnp_name_fn_t set_hnp_name; prte_plm_base_module_spawn_fn_t spawn; prte_plm_base_module_remote_spawn_fn_t remote_spawn; prte_plm_base_module_terminate_job_fn_t terminate_job; prte_plm_base_module_terminate_orteds_fn_t terminate_orteds; prte_plm_base_module_terminate_procs_fn_t terminate_procs; prte_plm_base_module_signal_job_fn_t signal_job; prte_plm_base_module_finalize_fn_t finalize; }; /** shprten prte_plm_base_module_1_0_0_t declaration */ typedef struct prte_plm_base_module_1_0_0_t prte_plm_base_module_1_0_0_t; /** shprten prte_plm_base_module_t declaration */ typedef struct prte_plm_base_module_1_0_0_t prte_plm_base_module_t; /** * plm component */ typedef pmix_mca_base_component_t prte_plm_base_component_t; /** * Macro for use in modules that are of type plm */ #define PRTE_PLM_BASE_VERSION_2_0_0 PRTE_MCA_BASE_VERSION_3_0_0("plm", 2, 0, 0) /* Global structure for accessing PLM functions */ PRTE_EXPORT extern prte_plm_base_module_t prte_plm; /* holds selected module's function pointers */ END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/plm/plm_types.h0000664000175000017500000003071315203542132020147 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PLM_TYPES_H #define PRTE_PLM_TYPES_H #include "prte_config.h" #include "types.h" BEGIN_C_DECLS /* * Process exit codes */ typedef int32_t prte_exit_code_t; /* * Process state codes */ #define PRTE_PROC_STATE_ANY 0xffff #define PRTE_PROC_STATE_UNDEF 0 /* undefined process state */ #define PRTE_PROC_STATE_INIT 1 /* process entry has been created by rmaps */ #define PRTE_PROC_STATE_RESTART 2 /* the proc is ready for restart */ #define PRTE_PROC_STATE_TERMINATE 3 /* process is marked for termination */ #define PRTE_PROC_STATE_RUNNING 4 /* daemon has locally fork'd process */ #define PRTE_PROC_STATE_REGISTERED 5 /* proc registered sync */ #define PRTE_PROC_STATE_IOF_COMPLETE 6 /* io forwarding pipes have closed */ #define PRTE_PROC_STATE_WAITPID_FIRED 7 /* waitpid fired on process */ #define PRTE_PROC_STATE_MODEX_READY 8 /* all modex info has been stored */ #define PRTE_PROC_STATE_READY_FOR_DEBUG 9 /* ready for debug */ /* * Define a "boundary" so we can easily and quickly determine * if a proc is still running or not - any value less than * this one means that we are not terminated */ #define PRTE_PROC_STATE_UNTERMINATED 15 #define PRTE_PROC_STATE_TERMINATED 20 /* process has terminated and is no longer running */ /* Define a boundary so we can easily and quickly determine * if a proc abnormally terminated - leave a little room * for future expansion */ #define PRTE_PROC_STATE_ERROR 50 /* Define specific error code values */ #define PRTE_PROC_STATE_KILLED_BY_CMD (PRTE_PROC_STATE_ERROR + 1) /* process was killed by PRTE cmd */ #define PRTE_PROC_STATE_ABORTED (PRTE_PROC_STATE_ERROR + 2) /* process aborted */ #define PRTE_PROC_STATE_FAILED_TO_START (PRTE_PROC_STATE_ERROR + 3) /* process failed to start */ #define PRTE_PROC_STATE_ABORTED_BY_SIG (PRTE_PROC_STATE_ERROR + 4) /* process aborted by signal */ #define PRTE_PROC_STATE_TERM_WO_SYNC (PRTE_PROC_STATE_ERROR + 5) /* process exit'd w/o required sync */ #define PRTE_PROC_STATE_COMM_FAILED (PRTE_PROC_STATE_ERROR + 6) /* process communication has failed */ #define PRTE_PROC_STATE_SENSOR_BOUND_EXCEEDED (PRTE_PROC_STATE_ERROR + 7) /* process exceeded a sensor limit */ #define PRTE_PROC_STATE_CALLED_ABORT (PRTE_PROC_STATE_ERROR + 8) /* process called "errmgr.abort" \ */ #define PRTE_PROC_STATE_HEARTBEAT_FAILED (PRTE_PROC_STATE_ERROR + 9) /* heartbeat failed to arrive */ #define PRTE_PROC_STATE_MIGRATING (PRTE_PROC_STATE_ERROR + 10) /* process failed and is waiting for resources before restarting \ */ #define PRTE_PROC_STATE_CANNOT_RESTART (PRTE_PROC_STATE_ERROR + 11) /* process failed and cannot be restarted */ #define PRTE_PROC_STATE_TERM_NON_ZERO (PRTE_PROC_STATE_ERROR + 12) /* process exited with a non-zero status, indicating abnormal */ #define PRTE_PROC_STATE_FAILED_TO_LAUNCH (PRTE_PROC_STATE_ERROR + 13) /* unable to launch process \ */ #define PRTE_PROC_STATE_UNABLE_TO_SEND_MSG (PRTE_PROC_STATE_ERROR + 14) /* unable to send a message */ #define PRTE_PROC_STATE_LIFELINE_LOST (PRTE_PROC_STATE_ERROR + 15) /* connection to lifeline lost \ */ #define PRTE_PROC_STATE_NO_PATH_TO_TARGET (PRTE_PROC_STATE_ERROR + 16) /* no path for communicating to target peer */ #define PRTE_PROC_STATE_FAILED_TO_CONNECT (PRTE_PROC_STATE_ERROR + 17) /* unable to connect to target peer */ #define PRTE_PROC_STATE_PEER_UNKNOWN (PRTE_PROC_STATE_ERROR + 18) /* unknown peer */ /* Define a boundary so that external developers * have a starting point for defining their own * proc states */ #define PRTE_PROC_STATE_DYNAMIC 100 /* * App_context state codes */ typedef int32_t prte_app_state_t; #define PRTE_APP_STATE_UNDEF 0 #define PRTE_APP_STATE_INIT 1 #define PRTE_APP_STATE_ALL_MAPPED 2 #define PRTE_APP_STATE_RUNNING 3 #define PRTE_APP_STATE_COMPLETED 4 /* * Job state codes */ typedef int32_t prte_job_state_t; #define PRTE_JOB_STATE_ANY INT_MAX #define PRTE_JOB_STATE_UNDEF 0 #define PRTE_JOB_STATE_INIT 1 /* ready to be assigned id */ #define PRTE_JOB_STATE_INIT_COMPLETE 2 /* jobid assigned and setup */ #define PRTE_JOB_STATE_ALLOCATE 3 /* ready to be allocated */ #define PRTE_JOB_STATE_ALLOCATION_COMPLETE 4 /* allocation completed */ #define PRTE_JOB_STATE_MAP 5 /* ready to be mapped */ #define PRTE_JOB_STATE_MAP_COMPLETE 6 /* mapping complete */ #define PRTE_JOB_STATE_SYSTEM_PREP 7 /* ready for final sanity check and system values updated */ #define PRTE_JOB_STATE_LAUNCH_DAEMONS 8 /* ready to launch daemons */ #define PRTE_JOB_STATE_DAEMONS_LAUNCHED 9 /* daemons for this job have been launched */ #define PRTE_JOB_STATE_DAEMONS_REPORTED 10 /* all launched daemons have reported */ #define PRTE_JOB_STATE_VM_READY 11 /* the VM is ready for operation */ #define PRTE_JOB_STATE_LAUNCH_APPS 12 /* ready to launch apps */ #define PRTE_JOB_STATE_SEND_LAUNCH_MSG 13 /* send launch msg to daemons */ #define PRTE_JOB_STATE_RUNNING 14 /* all procs have been fork'd */ #define PRTE_JOB_STATE_SUSPENDED 15 /* job has been suspended */ #define PRTE_JOB_STATE_REGISTERED 16 /* all procs registered for sync */ #define PRTE_JOB_STATE_LOCAL_LAUNCH_COMPLETE 18 /* all local procs have attempted launch */ #define PRTE_JOB_STATE_READY_FOR_DEBUG 19 /* all local procs report ready for debug */ #define PRTE_JOB_STATE_STARTED 20 /* first process has been started */ /* * Define a "boundary" so we can easily and quickly determine * if a job is still running or not - any value less than * this one means that we are not terminated */ #define PRTE_JOB_STATE_UNTERMINATED 30 #define PRTE_JOB_STATE_TERMINATED 31 /* all processes have terminated and job is no longer running */ #define PRTE_JOB_STATE_ALL_JOBS_COMPLETE 32 #define PRTE_JOB_STATE_DAEMONS_TERMINATED 33 #define PRTE_JOB_STATE_NOTIFY_COMPLETED 34 /* callback to notify when job completes */ #define PRTE_JOB_STATE_NOTIFIED 35 /* Define a boundary so we can easily and quickly determine * if a job abnormally terminated - leave a little room * for future expansion */ #define PRTE_JOB_STATE_ERROR 50 /* Define specific error code values */ #define PRTE_JOB_STATE_KILLED_BY_CMD (PRTE_JOB_STATE_ERROR + 1) /* job was killed by PRTE cmd */ #define PRTE_JOB_STATE_ABORTED (PRTE_JOB_STATE_ERROR + 2) /* at least one process aborted, causing job to abort */ #define PRTE_JOB_STATE_FAILED_TO_START (PRTE_JOB_STATE_ERROR + 3) /* at least one process failed to start */ #define PRTE_JOB_STATE_ABORTED_BY_SIG (PRTE_JOB_STATE_ERROR + 4) /* job was killed by a signal */ #define PRTE_JOB_STATE_ABORTED_WO_SYNC (PRTE_JOB_STATE_ERROR + 5) /* job was aborted because proc exit'd w/o required sync */ #define PRTE_JOB_STATE_COMM_FAILED (PRTE_JOB_STATE_ERROR + 6) /* communication has failed */ #define PRTE_JOB_STATE_SENSOR_BOUND_EXCEEDED (PRTE_JOB_STATE_ERROR + 7) /* job had a process that exceeded a sensor limit */ #define PRTE_JOB_STATE_CALLED_ABORT (PRTE_JOB_STATE_ERROR + 8) /* at least one process called "errmgr.abort" */ #define PRTE_JOB_STATE_HEARTBEAT_FAILED (PRTE_JOB_STATE_ERROR + 9) /* heartbeat failed to arrive \ */ #define PRTE_JOB_STATE_NEVER_LAUNCHED (PRTE_JOB_STATE_ERROR + 10) /* the job never even attempted to launch due to \ * an error earlier in the \ * launch procedure \ */ #define PRTE_JOB_STATE_ABORT_ORDERED (PRTE_JOB_STATE_ERROR + 11) /* the processes in this job have been ordered to "die", \ * but may not have completed it yet. Don't order it again \ */ #define PRTE_JOB_STATE_NON_ZERO_TERM (PRTE_JOB_STATE_ERROR + 12) /* at least one process exited with non-zero status */ #define PRTE_JOB_STATE_FAILED_TO_LAUNCH (PRTE_JOB_STATE_ERROR + 13) #define PRTE_JOB_STATE_FORCED_EXIT (PRTE_JOB_STATE_ERROR + 14) #define PRTE_JOB_STATE_SILENT_ABORT (PRTE_JOB_STATE_ERROR + 16) /* an error occurred and was reported elsewhere, so error out quietly */ #define PRTE_JOB_STATE_REPORT_PROGRESS (PRTE_JOB_STATE_ERROR + 17) /* report launch progress - not an error */ #define PRTE_JOB_STATE_ALLOC_FAILED (PRTE_JOB_STATE_ERROR + 18) /* job failed to obtain an allocation */ #define PRTE_JOB_STATE_MAP_FAILED (PRTE_JOB_STATE_ERROR + 19) /* job failed to map */ #define PRTE_JOB_STATE_CANNOT_LAUNCH (PRTE_JOB_STATE_ERROR + 20) /* resources were busy and so the job cannot be launched */ #define PRTE_JOB_STATE_FILES_POSN_FAILED (PRTE_JOB_STATE_ERROR + 21) #define PRTE_JOB_STATE_FT (PRTE_JOB_STATE_ERROR + 200) /* define an FT event */ #define PRTE_JOB_STATE_FT_CHECKPOINT (PRTE_JOB_STATE_FT + 1) #define PRTE_JOB_STATE_FT_CONTINUE (PRTE_JOB_STATE_FT + 2) #define PRTE_JOB_STATE_FT_RESTART (PRTE_JOB_STATE_FT + 3) /* Define a boundary so that external developers * have a starting point for defining their own * job states */ #define PRTE_JOB_STATE_DYNAMIC (PRTE_JOB_STATE_ERROR + 2000) /** * Node State, corresponding to the PRTE_NODE_STATE_* #defines, * below. These are #defines instead of an enum because the thought * is that we may have lots and lots of entries of these in the * registry and by making this an int8_t, it's only 1 byte, whereas an * enum defaults to an int (probably 4 bytes). So it's a bit of a * space savings. */ typedef int8_t prte_node_state_t; #define PRTE_NODE_STATE_UNDEF 0 // Node is undefined #define PRTE_NODE_STATE_UNKNOWN 1 // Node is defined but in an unknown state #define PRTE_NODE_STATE_DOWN 2 // Node is down #define PRTE_NODE_STATE_UP 3 // Node is up / available for use #define PRTE_NODE_STATE_REBOOT 4 // Node is rebooting #define PRTE_NODE_STATE_DO_NOT_USE 5 // Node is up, but not available for use for the next mapping #define PRTE_NODE_STATE_NOT_INCLUDED 6 // Node is up, but not part of the node pool for jobs #define PRTE_NODE_STATE_ADDED 7 // Node was dynamically added to pool /* Define a boundary so that external developers * have a starting point for defining their own * node states */ #define PRTE_NODE_STATE_DYNAMIC 100 /* * PLM commands */ typedef uint8_t prte_plm_cmd_flag_t; #define PRTE_PLM_LAUNCH_JOB_CMD 1 #define PRTE_PLM_UPDATE_PROC_STATE 2 #define PRTE_PLM_REGISTERED_CMD 3 #define PRTE_PLM_TOOL_ATTACHED_CMD 4 #define PRTE_PLM_READY_FOR_DEBUG_CMD 5 #define PRTE_PLM_LOCAL_LAUNCH_COMP_CMD 6 END_C_DECLS #endif prrte-4.1.1~rc2/src/mca/plm/lsf/0000775000175000017500000000000015203542132016542 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/plm/lsf/owner.txt0000664000175000017500000000024415203542132020435 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/plm/lsf/configure.m40000664000175000017500000000315315203542132020767 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2019 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_plm_lsf_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_plm_lsf_CONFIG],[ AC_CONFIG_FILES([src/mca/plm/lsf/Makefile]) PRTE_CHECK_LSF([plm_lsf], [plm_lsf_good=1], [plm_lsf_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$plm_lsf_good" = "1" || test "$prte_testbuild_launchers" = "1"], [$1], [$2]) # set build flags to use in makefile AC_SUBST([plm_lsf_CPPFLAGS]) AC_SUBST([plm_lsf_LDFLAGS]) AC_SUBST([plm_lsf_LIBS]) ])dnl prrte-4.1.1~rc2/src/mca/plm/lsf/plm_lsf_component.c0000664000175000017500000000753715203542132022440 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2008 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #if PRTE_TESTBUILD_LAUNCHERS #include "testbuild_lsf.h" #else #include #endif #include "src/util/pmix_output.h" #include "plm_lsf.h" #include "src/mca/plm/base/base.h" #include "src/mca/plm/base/plm_private.h" #include "src/mca/plm/plm.h" /* * Public string showing the plm lsf component version number */ const char *prte_mca_plm_lsf_component_version_string = "PRTE lsf plm MCA component version " PRTE_VERSION; /* * Local function */ static int plm_lsf_open(void); static int plm_lsf_close(void); static int prte_mca_plm_lsf_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_mca_plm_lsf_component_t prte_mca_plm_lsf_component = { .super = { PRTE_PLM_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "lsf", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = plm_lsf_open, .pmix_mca_close_component = plm_lsf_close, .pmix_mca_query_component = prte_mca_plm_lsf_component_query, } }; PMIX_MCA_BASE_COMPONENT_INIT(prte, plm, lsf) static int plm_lsf_open(void) { return PRTE_SUCCESS; } static int plm_lsf_close(void) { return PRTE_SUCCESS; } static int prte_mca_plm_lsf_component_query(pmix_mca_base_module_t **module, int *priority) { /* check if lsf is running here and make sure IBM CSM is NOT enabled */ if (NULL == getenv("LSB_JOBID") || getenv("CSM_ALLOCATION_ID") || lsb_init("PRTE launcher") < 0) { /* nope, not here */ pmix_output_verbose(10, prte_plm_base_framework.framework_output, "plm:lsf: NOT available for selection"); *module = NULL; return PRTE_ERROR; } *priority = 75; *module = (pmix_mca_base_module_t *) &prte_plm_lsf_module; return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/plm/lsf/plm_lsf.h0000664000175000017500000000305715203542132020354 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022 Nanook Consulting. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PLM_LSF_H #define PRTE_PLM_LSF_H #include "prte_config.h" #include "src/mca/mca.h" #include "src/mca/plm/plm.h" BEGIN_C_DECLS struct prte_mca_plm_lsf_component_t { prte_plm_base_component_t super; bool timing; }; typedef struct prte_mca_plm_lsf_component_t prte_mca_plm_lsf_component_t; /* Globally exported variables */ PRTE_EXPORT extern prte_mca_plm_lsf_component_t prte_mca_plm_lsf_component; extern prte_plm_base_module_t prte_plm_lsf_module; END_C_DECLS #endif /* PRTE_PLM_LSFH */ prrte-4.1.1~rc2/src/mca/plm/lsf/testbuild_lsf.h0000664000175000017500000000272015203542132021557 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2022-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PLM_LSF_TESTBUILD_H #define PRTE_PLM_LSF_TESTBUILD_H #include "prte_config.h" #include "src/mca/mca.h" #include "src/mca/plm/plm.h" BEGIN_C_DECLS #define LSF_DJOB_REPLACE_ENV 0x1 #define LSF_DJOB_NOWAIT 0x2 int lsb_init(char *str); int lsb_launch(char **nodelist_argv, char **argv, int flags, char **env); char *lsb_sysmsg(void); extern int lsberrno; END_C_DECLS #endif /* PRTE_PLM_LSF_TESTBUILD_H */ prrte-4.1.1~rc2/src/mca/plm/lsf/Makefile.am0000664000175000017500000000407615203542132020605 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2008 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2008 Institut National de Recherche en Informatique # et Automatique. All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # AM_CPPFLAGS = $(plm_lsf_CPPFLAGS) EXTRA_DIST = \ help-plm-lsf.txt \ testbuild_lsf.h sources = \ plm_lsf.h \ plm_lsf_component.c \ plm_lsf_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_plm_lsf_DSO component_noinst = component_install = prte_mca_plm_lsf.la else component_noinst = libprtemca_plm_lsf.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_plm_lsf_la_SOURCES = $(sources) prte_mca_plm_lsf_la_LDFLAGS = -module -avoid-version $(plm_lsf_LDFLAGS) prte_mca_plm_lsf_la_LIBADD = $(plm_lsf_LIBS) \ $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_plm_lsf_la_SOURCES = $(sources) libprtemca_plm_lsf_la_LDFLAGS = -module -avoid-version $(plm_lsf_LDFLAGS) libprtemca_plm_lsf_la_LIBADD = $(plm_lsf_LIBS) prrte-4.1.1~rc2/src/mca/plm/lsf/plm_lsf_module.c0000664000175000017500000003653515203542132021723 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2008 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2007-2012 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2008 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include "types.h" #include #ifdef HAVE_UNISTD_H # include #endif #include #include #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_FCNTL_H # include #endif #define SR1_PJOBS #if PRTE_TESTBUILD_LAUNCHERS #include "testbuild_lsf.h" int lsberrno; #else #include #endif #include "src/pmix/pmix-internal.h" #include "src/mca/base/pmix_base.h" #include "src/mca/prteinstalldirs/prteinstalldirs.h" #include "src/mca/pinstalldirs/pinstalldirs_types.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_output.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_printf.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/rmaps.h" #include "src/mca/schizo/schizo.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/pmix_show_help.h" #include "plm_lsf.h" #include "src/mca/plm/base/base.h" #include "src/mca/plm/base/plm_private.h" #include "src/mca/plm/plm.h" /* * Local functions */ static int plm_lsf_init(void); static int plm_lsf_launch_job(prte_job_t *jdata); static int plm_lsf_terminate_orteds(void); static int plm_lsf_signal_job(pmix_nspace_t jobid, int32_t signal); static int plm_lsf_finalize(void); /* * Global variable */ prte_plm_base_module_t prte_plm_lsf_module = { .init = plm_lsf_init, .set_hnp_name = prte_plm_base_set_hnp_name, .spawn = plm_lsf_launch_job, .terminate_job = prte_plm_base_prted_terminate_job, .terminate_orteds = plm_lsf_terminate_orteds, .terminate_procs = prte_plm_base_prted_kill_local_procs, .signal_job = plm_lsf_signal_job, .finalize = plm_lsf_finalize }; static void launch_daemons(int fd, short args, void *cbdata); /** * Init the module */ int plm_lsf_init(void) { int rc; prte_job_t *daemons; if (PRTE_SUCCESS != (rc = prte_plm_base_comm_start())) { PRTE_ERROR_LOG(rc); } daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); if (prte_get_attribute(&daemons->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* must assign daemons as won't be launching them */ prte_plm_globals.daemon_nodes_assigned_at_launch = true; } else { /* we do NOT assign daemons to nodes at launch - we will * determine that mapping when the daemon * calls back. This is required because lsf does * its own mapping of proc-to-node, and we cannot know * in advance which daemon will wind up on which node */ prte_plm_globals.daemon_nodes_assigned_at_launch = false; } /* point to our launch command */ if (PRTE_SUCCESS != (rc = prte_state.add_job_state(PRTE_JOB_STATE_LAUNCH_DAEMONS, launch_daemons))) { PRTE_ERROR_LOG(rc); return rc; } return rc; } /* When working in this function, ALWAYS jump to "cleanup" if * you encounter an error so that prun will be woken up and * the job can cleanly terminate */ static int plm_lsf_launch_job(prte_job_t *jdata) { if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { /* this is a restart situation - skip to the mapping stage */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP); } else { /* new job - set it up */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_INIT); } return PRTE_SUCCESS; } static void launch_daemons(int fd, short args, void *cbdata) { prte_job_map_t *map; size_t num_nodes; char *param; char **argv = NULL; int argc; int rc; char **env = NULL; char **nodelist_argv; int nodelist_argc; char *vpid_string; char *cur_prefix; int proc_vpid_index = 0; bool failed_launch = true; prte_node_t *node; int32_t nnode; prte_job_t *daemons; prte_state_caddy_t *state = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; char *pmix_prefix = NULL; char *newenv; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(state); jdata = state->jdata; /* start by setting up the virtual machine */ daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); if (PRTE_SUCCESS != (rc = prte_plm_base_setup_virtual_machine(jdata))) { PRTE_ERROR_LOG(rc); goto cleanup; } /* if we don't want to launch, then don't attempt to * launch the daemons - the user really wants to just * look at the proposed process map */ if (prte_get_attribute(&daemons->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* set the state to indicate the daemons reported - this * will trigger the daemons_reported event and cause the * job to move to the following step */ state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); PMIX_RELEASE(state); return; } PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:lsf: launching vm", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* Get the map for this job */ if (NULL == (map = daemons->map)) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto cleanup; } num_nodes = map->num_new_daemons; if (0 == num_nodes) { /* set the state to indicate the daemons reported - this * will trigger the daemons_reported event and cause the * job to move to the following step */ PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:lsf: no new daemons to launch", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); PMIX_RELEASE(state); return; } /* create nodelist */ nodelist_argv = NULL; nodelist_argc = 0; for (nnode = 0; nnode < map->nodes->size; nnode++) { if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, nnode))) { continue; } /* if the daemon already exists on this node, then * don't include it */ if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_DAEMON_LAUNCHED)) { continue; } /* otherwise, add it to the list of nodes upon which * we need to launch a daemon */ pmix_argv_append(&nodelist_argc, &nodelist_argv, node->name); } /* * start building argv array */ argv = NULL; argc = 0; /* * PRTED OPTIONS */ /* protect against launchers that forward the entire environment */ if (NULL != getenv("PMIX_LAUNCHER_PAUSE_FOR_TOOL")) { unsetenv("PMIX_LAUNCHER_PAUSE_FOR_TOOL"); } if (NULL != getenv("PMIX_LAUNCHER_RENDEZVOUS_FILE")) { unsetenv("PMIX_LAUNCHER_RENDEZVOUS_FILE"); } /* add the daemon command (as specified by user) */ prte_plm_base_setup_prted_cmd(&argc, &argv); /* Add basic orted command line options */ prte_plm_base_prted_append_basic_args(&argc, &argv, "lsf", &proc_vpid_index); /* tell the new daemons the base of the name list so they can compute * their own name on the other end */ rc = prte_util_convert_vpid_to_string(&vpid_string, map->daemon_vpid_start); if (PRTE_SUCCESS != rc) { pmix_output(0, "plm_lsf: unable to get daemon vpid as string"); goto cleanup; } free(argv[proc_vpid_index]); argv[proc_vpid_index] = strdup(vpid_string); free(vpid_string); /* protect the args in case someone has a script wrapper */ prte_plm_base_wrap_args(argv); if (0 < pmix_output_get_verbosity(prte_plm_base_framework.framework_output)) { param = PMIx_Argv_join(argv, ' '); if (NULL != param) { pmix_output(0, "plm:lsf: final top-level argv:"); pmix_output(0, "plm:lsf: %s", param); free(param); } } /* setup environment - this is the pristine version that PRRTE * has already stripped of all PRTE_ and PMIX_ prefixed values */ env = PMIx_Argv_copy(prte_launch_environ); /* * Any prefix was installed in the DAEMON job object, so * we only need to look there to find it. This covers any * prefix by default, PRTE_PREFIX given in the environment, * and '--prefix' from the cmd line */ if (prte_get_attribute(&daemons->attributes, PRTE_JOB_PREFIX, (void **) &cur_prefix, PMIX_STRING)) { char *bin_base = pmix_basename(prte_install_dirs.bindir); char *lib_base = pmix_basename(prte_install_dirs.libdir); // set the path param = getenv("PATH"); if (NULL != param) { pmix_asprintf(&newenv, "%s/%s:%s", cur_prefix, bin_base, param); } else { pmix_asprintf(&newenv, "%s/%s", cur_prefix, bin_base); } PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:lsf: resetting PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); PMIx_Setenv("PATH", newenv, true, &env); free(newenv); // set the library path param = getenv("LD_LIBRARY_PATH"); if (NULL != param) { pmix_asprintf(&newenv, "%s/%s:%s", cur_prefix, lib_base, param); } else { pmix_asprintf(&newenv, "%s/%s", cur_prefix, lib_base); } PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:lsf: resetting LD_LIBRARY_PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); PMIx_Setenv("LD_LIBRARY_PATH", newenv, true, &env); free(newenv); // add the prefix itself to the environment PMIx_Setenv("PRTE_PREFIX", cur_prefix, true, &env); free(cur_prefix); free(bin_base); free(lib_base); } /* Similarly, we have to check for any PMIx prefix that was specified */ if (prte_get_attribute(&daemons->attributes, PRTE_JOB_PMIX_PREFIX, (void **) &pmix_prefix, PMIX_STRING)) { char *lib_base = pmix_basename(pmix_pinstall_dirs.libdir); // set the library path param = getenv("LD_LIBRARY_PATH"); if (NULL != param) { pmix_asprintf(&newenv, "%s/%s:%s", pmix_prefix, lib_base, param); } else { pmix_asprintf(&newenv, "%s/%s", pmix_prefix, lib_base); } PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:lsf: resetting LD_LIBRARY_PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); PMIx_Setenv("LD_LIBRARY_PATH", newenv, true, &env); free(newenv); // add the prefix itself to the environment PMIx_Setenv("PMIX_PREFIX", pmix_prefix, true, &env); free(pmix_prefix); free(lib_base); } /* lsb_launch tampers with SIGCHLD. * After the call to lsb_launch, the signal handler for SIGCHLD is NULL. * So, we disable the SIGCHLD handler of libevent for the duration of * the call to lsb_launch */ prte_wait_disable(); /* exec the daemon(s). Do NOT wait for lsb_launch to complete as * it only completes when the processes it starts - in this case, * the orteds - complete. We need to go ahead and return so * prun can do the rest of its stuff. Instead, we'll catch any * failures and deal with them elsewhere */ if ((rc = lsb_launch(nodelist_argv, argv, LSF_DJOB_REPLACE_ENV | LSF_DJOB_NOWAIT, env)) < 0) { PRTE_ERROR_LOG(PRTE_ERR_FAILED_TO_START); char *flattened_nodelist = NULL; flattened_nodelist = PMIx_Argv_join(nodelist_argv, '\n'); pmix_show_help("help-plm-lsf.txt", "lsb_launch-failed", true, rc, lsberrno, lsb_sysmsg(), PMIx_Argv_count(nodelist_argv), flattened_nodelist); free(flattened_nodelist); rc = PRTE_ERR_FAILED_TO_START; prte_wait_enable(); /* re-enable our SIGCHLD handler */ goto cleanup; } prte_wait_enable(); /* re-enable our SIGCHLD handler */ /* indicate that the daemons for this job were launched */ state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; daemons->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; /* flag that launch was successful, so far as we currently know */ failed_launch = false; cleanup: if (NULL != argv) { PMIx_Argv_free(argv); } if (NULL != env) { PMIx_Argv_free(env); } /* check for failed launch - if so, force terminate */ if (failed_launch) { PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_FAILED_TO_START); } /* cleanup the caddy */ PMIX_RELEASE(state); } /** * Terminate the orteds for a given job */ static int plm_lsf_terminate_orteds(void) { int rc; if (PRTE_SUCCESS != (rc = prte_plm_base_prted_exit(PRTE_DAEMON_EXIT_CMD))) { PRTE_ERROR_LOG(rc); } return rc; } /** * Signal all the processes in the job */ static int plm_lsf_signal_job(pmix_nspace_t jobid, int32_t signal) { int rc; /* order the orteds to pass this signal to their local procs */ if (PRTE_SUCCESS != (rc = prte_plm_base_prted_signal_local_procs(jobid, signal))) { PRTE_ERROR_LOG(rc); } return rc; } static int plm_lsf_finalize(void) { int rc; /* cleanup any pending recvs */ if (PRTE_SUCCESS != (rc = prte_plm_base_comm_stop())) { PRTE_ERROR_LOG(rc); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/plm/lsf/help-plm-lsf.txt0000664000175000017500000000241015203542132021600 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # [lsb_launch-failed] The LSF process starter (lsb_launch) failed to start the daemons on the nodes in the allocation. Returned : %d lsberrno : (%d) %s This may mean that one or more of the nodes in the LSF allocation is not setup properly. Below is a list of the %d nodes that were passed to lsb_launch: %s prrte-4.1.1~rc2/src/mca/plm/slurm/0000775000175000017500000000000015203542132017120 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/plm/slurm/plm_slurm_module.c0000664000175000017500000006143315203542132022652 0ustar alastairalastair/* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "src/runtime/prte_globals.h" #include #include #ifdef HAVE_UNISTD_H # include #endif #include #include #ifdef HAVE_SYS_TYPES_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif #ifdef HAVE_SYS_STAT_H # include #endif #ifdef HAVE_FCNTL_H # include #endif #include "src/mca/base/pmix_base.h" #include "src/mca/prteinstalldirs/prteinstalldirs.h" #include "src/mca/pinstalldirs/pinstalldirs_types.h" #include "src/util/pmix_argv.h" #include "src/util/pmix_basename.h" #include "src/util/pmix_output.h" #include "src/util/pmix_path.h" #include "src/util/pmix_environ.h" #include "constants.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/schizo/schizo.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_quit.h" #include "src/runtime/prte_wait.h" #include "src/threads/pmix_threads.h" #include "src/util/name_fns.h" #include "src/util/pmix_show_help.h" #include "types.h" #include "src/prted/prted.h" #include "plm_slurm.h" #include "src/mca/plm/base/base.h" #include "src/mca/plm/base/plm_private.h" #include "src/mca/plm/plm.h" /* * Local functions */ static int plm_slurm_init(void); static int plm_slurm_launch_job(prte_job_t *jdata); static int plm_slurm_terminate_prteds(void); static int plm_slurm_signal_job(pmix_nspace_t jobid, int32_t signal); static int plm_slurm_finalize(void); static int plm_slurm_start_proc(int argc, char **argv, char *prefix, char *pmix_prefix); /* * Global variable */ prte_plm_base_module_1_0_0_t prte_plm_slurm_module = { .init = plm_slurm_init, .set_hnp_name = prte_plm_base_set_hnp_name, .spawn = plm_slurm_launch_job, .terminate_job = prte_plm_base_prted_terminate_job, .terminate_orteds = plm_slurm_terminate_prteds, .terminate_procs = prte_plm_base_prted_kill_local_procs, .signal_job = plm_slurm_signal_job, .finalize = plm_slurm_finalize }; /* * Local variables */ static pid_t primary_srun_pid = 0; static bool primary_pid_set = false; static void launch_daemons(int fd, short args, void *cbdata); /** * Init the module */ static int plm_slurm_init(void) { int rc; prte_job_t *jdata; if (PRTE_SUCCESS != (rc = prte_plm_base_comm_start())) { PRTE_ERROR_LOG(rc); return rc; } /* if we don't want to launch (e.g., someone just wants * to test the mappers), then we assign vpids at "launch" * so the mapper has something to work with */ jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { prte_plm_globals.daemon_nodes_assigned_at_launch = true; } else { /* we do NOT assign daemons to nodes at launch - we will * determine that mapping when the daemon * calls back. This is required because slurm does * its own mapping of proc-to-node, and we cannot know * in advance which daemon will wind up on which node */ prte_plm_globals.daemon_nodes_assigned_at_launch = false; } /* point to our launch command */ if (PRTE_SUCCESS != (rc = prte_state.add_job_state(PRTE_JOB_STATE_LAUNCH_DAEMONS, launch_daemons))) { PRTE_ERROR_LOG(rc); return rc; } return rc; } /* When working in this function, ALWAYS jump to "cleanup" if * you encounter an error so that prun will be woken up and * the job can cleanly terminate */ static int plm_slurm_launch_job(prte_job_t *jdata) { if (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_RESTART)) { /* this is a restart situation - skip to the mapping stage */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP); } else { /* new job - set it up */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_INIT); } return PRTE_SUCCESS; } static void launch_daemons(int fd, short args, void *cbdata) { prte_node_t *node; int32_t n; prte_job_map_t *map; char *param; char **argv = NULL; int argc; int rc; char *tmp; char *nodelist_flat; char **nodelist_argv; char *name_string; char **custom_strings; int num_args, i; char *cur_prefix = NULL; char *pmix_prefix = NULL; int proc_vpid_index; bool failed_launch = true; prte_job_t *daemons; prte_state_caddy_t *state = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(state); PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: LAUNCH DAEMONS CALLED", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* start by setting up the virtual machine */ daemons = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); if (PRTE_SUCCESS != (rc = prte_plm_base_setup_virtual_machine(state->jdata))) { PRTE_ERROR_LOG(rc); goto cleanup; } /* if we don't want to launch, then don't attempt to * launch the daemons - the user really wants to just * look at the proposed process map */ if (prte_get_attribute(&daemons->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* set the state to indicate the daemons reported - this * will trigger the daemons_reported event and cause the * job to move to the following step */ state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); PMIX_RELEASE(state); return; } /* Get the map for this job */ if (NULL == (map = daemons->map)) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); rc = PRTE_ERR_NOT_FOUND; goto cleanup; } if (0 == map->num_new_daemons) { /* set the state to indicate the daemons reported - this * will trigger the daemons_reported event and cause the * job to move to the following step */ PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: no new daemons to launch", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); PMIX_RELEASE(state); return; } /* * start building argv array */ argv = NULL; argc = 0; /* * SLURM srun OPTIONS */ /* add the srun command */ pmix_argv_append(&argc, &argv, "srun"); // add the external launcher flag if necessary if (!prte_mca_plm_slurm_component.early) { pmix_argv_append(&argc, &argv, "--external-launcher"); } /* start one orted on each node */ pmix_argv_append(&argc, &argv, "--ntasks-per-node=1"); if (!prte_get_attribute(&state->jdata->attributes, PRTE_JOB_RECOVERABLE, NULL, PMIX_BOOL) && !prte_get_attribute(&state->jdata->attributes, PRTE_JOB_CONTINUOUS, NULL, PMIX_BOOL)) { /* kill the job if any prteds die */ pmix_argv_append(&argc, &argv, "--kill-on-bad-exit"); } /* our daemons are not an MPI task */ pmix_argv_append(&argc, &argv, "--mpi=none"); /* ensure the orteds are not bound to a single processor, * just in case the TaskAffinity option is set by default. * This will *not* release the orteds from any cpu-set * constraint, but will ensure it doesn't get * bound to only one processor */ pmix_argv_append(&argc, &argv, "--cpu-bind=none"); /* Append user defined arguments to srun */ if (NULL != prte_mca_plm_slurm_component.custom_args) { custom_strings = PMIx_Argv_split(prte_mca_plm_slurm_component.custom_args, ' '); num_args = PMIx_Argv_count(custom_strings); for (i = 0; i < num_args; ++i) { pmix_argv_append(&argc, &argv, custom_strings[i]); } PMIx_Argv_free(custom_strings); } /* create nodelist */ nodelist_argv = NULL; for (n = 0; n < map->nodes->size; n++) { if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(map->nodes, n))) { continue; } /* if the daemon already exists on this node, then * don't include it */ if (PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_DAEMON_LAUNCHED)) { continue; } /* otherwise, add it to the list of nodes upon which * we need to launch a daemon */ PMIx_Argv_append_nosize(&nodelist_argv, node->name); } if (0 == PMIx_Argv_count(nodelist_argv)) { pmix_show_help("help-plm-slurm.txt", "no-hosts-in-list", true); rc = PRTE_ERR_FAILED_TO_START; goto cleanup; } nodelist_flat = PMIx_Argv_join(nodelist_argv, ','); PMIx_Argv_free(nodelist_argv); /* if we are using all allocated nodes, then srun doesn't * require any further arguments */ if (map->num_new_daemons < prte_num_allocated_nodes) { pmix_asprintf(&tmp, "--nodes=%lu", (unsigned long) map->num_new_daemons); pmix_argv_append(&argc, &argv, tmp); free(tmp); pmix_asprintf(&tmp, "--nodelist=%s", nodelist_flat); pmix_argv_append(&argc, &argv, tmp); free(tmp); } /* tell srun how many tasks to run */ pmix_asprintf(&tmp, "--ntasks=%lu", (unsigned long) map->num_new_daemons); pmix_argv_append(&argc, &argv, tmp); free(tmp); PMIX_OUTPUT_VERBOSE((2, prte_plm_base_framework.framework_output, "%s plm:slurm: launching on nodes %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), nodelist_flat)); free(nodelist_flat); /* * PRTED OPTIONS */ /* add the daemon command (as specified by user) */ prte_plm_base_setup_prted_cmd(&argc, &argv); /* Add basic orted command line options, including debug flags */ prte_plm_base_prted_append_basic_args(&argc, &argv, "slurm", &proc_vpid_index); /* tell the new daemons the base of the name list so they can compute * their own name on the other end */ rc = prte_util_convert_vpid_to_string(&name_string, map->daemon_vpid_start); if (PRTE_SUCCESS != rc) { pmix_output(0, "plm_slurm: unable to get daemon vpid as string"); goto cleanup; } free(argv[proc_vpid_index]); argv[proc_vpid_index] = strdup(name_string); free(name_string); /* * Any prefix was installed in the DAEMON job object, so * we only need to look there to find it. This covers any * prefix by default, PRTE_PREFIX given in the environment, * and '--prefix' from the cmd line */ if (!prte_get_attribute(&daemons->attributes, PRTE_JOB_PREFIX, (void **) &cur_prefix, PMIX_STRING)) { cur_prefix = NULL; } /* Similarly, we have to check for any PMIx prefix that was specified */ if (!prte_get_attribute(&daemons->attributes, PRTE_JOB_PMIX_PREFIX, (void **) &pmix_prefix, PMIX_STRING)) { pmix_prefix = NULL; } /* protect the args in case someone has a script wrapper around srun */ prte_plm_base_wrap_args(argv); if (0 < pmix_output_get_verbosity(prte_plm_base_framework.framework_output)) { param = PMIx_Argv_join(argv, ' '); pmix_output(prte_plm_base_framework.framework_output, "%s plm:slurm: final top-level argv:\n\t%s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), (NULL == param) ? "NULL" : param); if (NULL != param) free(param); } /* exec the daemon(s) */ if (PRTE_SUCCESS != (rc = plm_slurm_start_proc(argc, argv, cur_prefix, pmix_prefix))) { PRTE_ERROR_LOG(rc); goto cleanup; } /* indicate that the daemons for this job were launched */ state->jdata->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; daemons->state = PRTE_JOB_STATE_DAEMONS_LAUNCHED; /* flag that launch was successful, so far as we currently know */ failed_launch = false; cleanup: if (NULL != argv) { PMIx_Argv_free(argv); } if (NULL != cur_prefix) { free(cur_prefix); } if (NULL != pmix_prefix) { free(pmix_prefix); } /* check for failed launch - if so, force terminate */ if (failed_launch) { PRTE_ACTIVATE_JOB_STATE(state->jdata, PRTE_JOB_STATE_FAILED_TO_LAUNCH); } /* cleanup the caddy */ PMIX_RELEASE(state); } /** * Terminate the orteds for a given job */ static int plm_slurm_terminate_prteds(void) { int rc = PRTE_SUCCESS; prte_job_t *jdata; /* check to see if the primary pid is set. If not, this indicates * that we never launched any additional daemons, so we cannot * not wait for a waitpid to fire and tell us it's okay to * exit. Instead, we simply trigger an exit for ourselves */ if (primary_pid_set) { if (PRTE_SUCCESS != (rc = prte_plm_base_prted_exit(PRTE_DAEMON_EXIT_CMD))) { PRTE_ERROR_LOG(rc); } } else { PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: primary daemons complete!", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* need to set the #terminated value to avoid an incorrect error msg */ jdata->num_terminated = jdata->num_procs; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); } return rc; } /** * Signal all the processes in the child srun by sending the signal directly to it */ static int plm_slurm_signal_job(pmix_nspace_t jobid, int32_t signal) { int rc = PRTE_SUCCESS; /* order them to pass this signal to their local procs */ if (PRTE_SUCCESS != (rc = prte_plm_base_prted_signal_local_procs(jobid, signal))) { PRTE_ERROR_LOG(rc); } return rc; } static int plm_slurm_finalize(void) { int rc; /* cleanup any pending recvs */ if (PRTE_SUCCESS != (rc = prte_plm_base_comm_stop())) { PRTE_ERROR_LOG(rc); } return PRTE_SUCCESS; } static void srun_wait_cb(int sd, short fd, void *cbdata) { prte_wait_tracker_t *t2 = (prte_wait_tracker_t *) cbdata; prte_proc_t *proc = t2->child; prte_job_t *jdata; PRTE_HIDE_UNUSED_PARAMS(sd, fd); jdata = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* need to check that we are at least version 17.11 */ if (prte_mca_plm_slurm_component.ancient) { pmix_show_help("help-plm-slurm.txt", "ancient-version", true, prte_mca_plm_slurm_component.major, prte_mca_plm_slurm_component.minor); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); PMIX_RELEASE(t2); return; } /* According to the SLURM folks, srun always returns the highest exit code of our remote processes. Thus, a non-zero exit status doesn't necessarily mean that srun failed - it could be that an orted returned a non-zero exit status. Of course, that means the orted failed(!), so the end result is the same - the job didn't start. As a result, we really can't do much with the exit status itself - it could be something in errno (if srun itself failed), or it could be something returned by an orted, or it could be something returned by the OS (e.g., couldn't find the orted binary). Somebody is welcome to sort out all the options and pretty-print a better error message. For now, though, the only thing that really matters is that srun failed. Report the error and make sure that prun wakes up - otherwise, do nothing! Unfortunately, the pid returned here is the srun pid, not the pid of the proc that actually died! So, to avoid confusion, just use -1 as the pid so nobody thinks this is real */ /* abort only if the status returned is non-zero - i.e., if * the orteds exited with an error */ if (0 != proc->exit_code) { /* an orted must have died unexpectedly - report * that the daemon has failed so we exit */ pmix_show_help("help-plm-slurm.txt", "srun-failed", true, proc->exit_code); PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); } else { /* otherwise, check to see if this is the primary pid */ if (primary_srun_pid == proc->pid) { /* in this case, we just want to fire the proper trigger so * mpirun can exit */ PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: primary daemons complete!", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); /* need to set the #terminated value to avoid an incorrect error msg */ jdata->num_terminated = jdata->num_procs; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_TERMINATED); } } /* done with this dummy */ PMIX_RELEASE(t2); } static int plm_slurm_start_proc(int argc, char **argv, char *prefix, char *pmix_prefix) { int fd; int srun_pid; int n; char **tmp = NULL, *p; char *exec_argv = pmix_path_findv(argv[0], 0, environ, NULL); prte_proc_t *dummy; char *oldenv, *newenv; PRTE_HIDE_UNUSED_PARAMS(argc); if (NULL == exec_argv) { pmix_show_help("help-plm-slurm.txt", "no-srun", true); return PRTE_ERR_SILENT; } srun_pid = fork(); if (-1 == srun_pid) { PRTE_ERROR_LOG(PRTE_ERR_SYS_LIMITS_CHILDREN); free(exec_argv); return PRTE_ERR_SYS_LIMITS_CHILDREN; } /* if this is the primary launch - i.e., not a comm_spawn of a * child job - then save the pid */ if (0 < srun_pid && !primary_pid_set) { primary_srun_pid = srun_pid; primary_pid_set = true; } /* setup a dummy proc object to track the srun */ dummy = PMIX_NEW(prte_proc_t); dummy->pid = srun_pid; /* be sure to mark it as alive so we don't instantly fire */ PRTE_FLAG_SET(dummy, PRTE_PROC_FLAG_ALIVE); /* setup the waitpid so we can find out if srun succeeds! */ prte_wait_cb(dummy, srun_wait_cb, NULL); if (0 == srun_pid) { /* child */ char *bin_base = NULL, *lib_base = NULL; /* Slurm forwards the entire environment, which we * REALLY don't want them to do as it might contain * envars pertaining to tool connections. So purge * the environment of anything PMIx/PRRTE related - * we will have put anything we need on the cmd line */ for (n=0; NULL != environ[n]; n++) { if (0 == strncmp(environ[n], "PMIX_", 5) || 0 == strncmp(environ[n], "PRTE_", 5)) { PMIx_Argv_append_nosize(&tmp, environ[n]); } } if (NULL != tmp) { for (n=0; NULL != tmp[n]; n++) { p = strchr(tmp[n], '='); *p = '\0'; unsetenv(tmp[n]); } PMIx_Argv_free(tmp); } /* Figure out the basenames for the libdir and bindir. There is a lengthy comment about this in plm_rsh_module.c explaining all the rationale for how / why we're doing this. */ lib_base = pmix_basename(prte_install_dirs.libdir); bin_base = pmix_basename(prte_install_dirs.bindir); /* If we have a prefix, then modify the PATH and LD_LIBRARY_PATH environment variables. */ if (NULL != prefix) { /* Reset PATH */ oldenv = getenv("PATH"); if (NULL != oldenv) { pmix_asprintf(&newenv, "%s/%s:%s", prefix, bin_base, oldenv); } else { pmix_asprintf(&newenv, "%s/%s", prefix, bin_base); } PMIx_Setenv("PATH", newenv, true, &environ); PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: reset PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); free(newenv); /* Reset LD_LIBRARY_PATH */ oldenv = getenv("LD_LIBRARY_PATH"); if (NULL != oldenv) { pmix_asprintf(&newenv, "%s/%s:%s", prefix, lib_base, oldenv); } else { pmix_asprintf(&newenv, "%s/%s", prefix, lib_base); } PMIx_Setenv("LD_LIBRARY_PATH", newenv, true, &environ); PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: reset LD_LIBRARY_PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); free(newenv); // need to export it as well so srun will propagate it PMIx_Setenv("PRTE_PREFIX", prefix, true, &environ); } /* for pmix_prefix, we only have to modify the library path. * NOTE: obviously, we cannot know the lib_base used for the * PMIx library. All we can do is hope they used the same one * for PMIx as they did for the one they linked to PRRTE */ if (NULL != pmix_prefix) { oldenv = getenv("LD_LIBRARY_PATH"); p = pmix_basename(pmix_pinstall_dirs.libdir); if (NULL != oldenv) { pmix_asprintf(&newenv, "%s/%s:%s", pmix_prefix, p, oldenv); } else { pmix_asprintf(&newenv, "%s/%s", pmix_prefix, p); } free(p); PMIx_Setenv("LD_LIBRARY_PATH", newenv, true, &environ); PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: reset LD_LIBRARY_PATH: %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), newenv)); free(newenv); // need to export it as well so srun will propagate it PMIx_Setenv("PMIX_PREFIX", pmix_prefix, true, &environ); } fd = open("/dev/null", O_CREAT | O_RDWR | O_TRUNC, 0666); if (fd >= 0) { dup2(fd, 0); /* When not in debug mode and --debug-daemons was not passed, * tie stdout/stderr to dev null so we don't see messages from orted * EXCEPT if the user has requested that we leave sessions attached */ if (0 > pmix_output_get_verbosity(prte_plm_base_framework.framework_output) && !prte_debug_daemons_flag && !prte_leave_session_attached) { dup2(fd, 1); dup2(fd, 2); } /* Don't leave the extra fd to /dev/null open */ if (fd > 2) { close(fd); } } /* get the srun process out of prun's process group so that signals sent from the shell (like those resulting from cntl-c) don't get sent to srun */ setpgid(0, 0); execvp(exec_argv, argv); pmix_output(0, "plm:slurm:start_proc: exec failed"); /* don't return - need to exit - returning would be bad - we're not in the calling process anymore */ exit(1); } else { /* parent */ /* just in case, make sure that the srun process is not in our process group any more. Stevens says always do this on both sides of the fork... */ setpgid(srun_pid, srun_pid); free(exec_argv); } return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/plm/slurm/help-plm-slurm.txt0000664000175000017500000000476415203542132022552 0ustar alastairalastair# -*- text -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2014-2020 Intel, Inc. All rights reserved. # Copyright (c) 2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # [no-hosts-in-list] The SLURM process starter for PRTE didn't find any hosts in the map for this application. This can be caused by a lack of an allocation, or by an error in the PRTE code. Please check to ensure you have a SLURM allocation. If you do, then please pass the error to the PRTE user's mailing list for assistance. # [no-srun] The SLURM process starter for OpenMPI was unable to locate a usable "srun" command in its path. Please check your path and try again. # [ancient-version] The Slurm process starter requires a minimum Slurm version level of v17.11. The detected version being used here is: Major: %d Minor: %d SchedMD strongly recommends that you upgrade your Slurm version. Versions of Slurm prior to 21.08.8 and 20.11.9 have been found to have a security issue and are not supported - therefore, it is recommended that you upgrade to a level at or above those versions. It is recognized that upgrades can take some time to complete. In the interim, you can execute your job by ignoring the Slurm process starter via the following MCA parameter: Environment: PRTE_MCA_plm=^slurm Cmd line: --prtemca plm ^slurm Default MCA param file: plm = ^slurm This will result in use of the ssh process starter. This will have no impact on your application, but will result in any accounting being done solely at the allocation level instead of per-job. # [srun-failed] srun returned non-zero exit status (%d) from launching the per-node daemon. You may debug this problem further by augmenting the cmd line with: * "--debug-daemons" * "--leave-session-attached" * "--prtemca plm_base_verbose N" where N > 0 prrte-4.1.1~rc2/src/mca/plm/slurm/owner.txt0000664000175000017500000000024415203542132021013 0ustar alastairalastair# # owner/status file # owner: institution that is responsible for this package # status: e.g. active, maintenance, unmaintained # owner: INTEL status: maintenance prrte-4.1.1~rc2/src/mca/plm/slurm/plm_slurm_component.c0000664000175000017500000001416515203542132023367 0ustar alastairalastair/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2021-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * * These symbols are in a file by themselves to provide nice linker * semantics. Since linkers generally pull in symbols by object * files, keeping these symbols as the only symbols in this file * prevents utility programs such as "ompi_info" from having to import * entire components just to query their version and parameters. */ #include "prte_config.h" #include "constants.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/runtime/prte_globals.h" #include "src/util/name_fns.h" #include "src/util/pmix_environ.h" #include "src/util/pmix_show_help.h" #include "src/util/pmix_string_copy.h" #include "plm_slurm.h" #include "src/mca/plm/base/plm_private.h" #include "src/mca/plm/plm.h" /* * Public string showing the plm ompi_slurm component version number */ const char *prte_mca_plm_slurm_component_version_string = "PRTE slurm plm MCA component version " PRTE_VERSION; /* * Local functions */ static int plm_slurm_register(void); static int plm_slurm_open(void); static int plm_slurm_close(void); static int prte_mca_plm_slurm_component_query(pmix_mca_base_module_t **module, int *priority); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ prte_mca_plm_slurm_component_t prte_mca_plm_slurm_component = { .super = { PRTE_PLM_BASE_VERSION_2_0_0, /* Component name and version */ .pmix_mca_component_name = "slurm", PMIX_MCA_BASE_MAKE_VERSION(component, PRTE_MAJOR_VERSION, PRTE_MINOR_VERSION, PMIX_RELEASE_VERSION), /* Component open and close functions */ .pmix_mca_open_component = plm_slurm_open, .pmix_mca_close_component = plm_slurm_close, .pmix_mca_query_component = prte_mca_plm_slurm_component_query, .pmix_mca_register_component_params = plm_slurm_register, } /* Other prte_mca_plm_slurm_component_t items -- left uninitialized here; will be initialized in plm_slurm_open() */ }; PMIX_MCA_BASE_COMPONENT_INIT(prte, plm, slurm) static int plm_slurm_register(void) { pmix_mca_base_component_t *comp = &prte_mca_plm_slurm_component.super; prte_mca_plm_slurm_component.custom_args = NULL; pmix_mca_base_component_var_register(comp, "args", "Custom arguments to srun", PMIX_MCA_BASE_VAR_TYPE_STRING, &prte_mca_plm_slurm_component.custom_args); return PRTE_SUCCESS; } static int plm_slurm_open(void) { return PRTE_SUCCESS; } static int prte_mca_plm_slurm_component_query(pmix_mca_base_module_t **module, int *priority) { FILE *fp; char version[1024], *ptr; /* Are we running under a SLURM job? */ if (NULL != getenv("SLURM_JOBID")) { *priority = 75; PMIX_OUTPUT_VERBOSE((1, prte_plm_base_framework.framework_output, "%s plm:slurm: available for selection", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); // check the version fp = popen("srun --version", "r"); if (NULL == fp) { // cannot run srun, so we cannot support this job *module = NULL; return PRTE_ERROR; } if (NULL == fgets(version, sizeof(version), fp)) { pclose(fp); *module = NULL; return PRTE_ERROR; } pclose(fp); /* move cptr past the 1st space. if the line doesn't have a space, then ignore it */ ptr = strchr(version, ' '); if (NULL == ptr) { // play it safe *module = NULL; return PRTE_ERROR; } ++ptr; // parse on the dots prte_mca_plm_slurm_component.major = strtol(ptr, &ptr, 10); ++ptr; prte_mca_plm_slurm_component.minor = strtol(ptr, NULL, 10); if (23 > prte_mca_plm_slurm_component.major) { prte_mca_plm_slurm_component.early = true; } else if (23 < prte_mca_plm_slurm_component.major) { prte_mca_plm_slurm_component.early = false; } else if (11 > prte_mca_plm_slurm_component.minor) { prte_mca_plm_slurm_component.early = true; } else { prte_mca_plm_slurm_component.early = false; } // check for ancient if (prte_mca_plm_slurm_component.major < 17) { prte_mca_plm_slurm_component.ancient = true; } else if (17 == prte_mca_plm_slurm_component.major && prte_mca_plm_slurm_component.minor < 11) { prte_mca_plm_slurm_component.ancient = true; } else { prte_mca_plm_slurm_component.ancient = false; } *module = (pmix_mca_base_module_t *) &prte_plm_slurm_module; return PRTE_SUCCESS; } /* Sadly, no */ *module = NULL; return PRTE_ERROR; } static int plm_slurm_close(void) { return PRTE_SUCCESS; } prrte-4.1.1~rc2/src/mca/plm/slurm/plm_slurm.h0000664000175000017500000000326215203542132021306 0ustar alastairalastair/* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. * Copyright (c) 2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2022-2025 Nanook Consulting All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ */ #ifndef PRTE_PLM_SLURM_EXPORT_H #define PRTE_PLM_SLURM_EXPORT_H #include "prte_config.h" #include "src/mca/mca.h" #include "src/mca/plm/plm.h" BEGIN_C_DECLS struct prte_mca_plm_slurm_component_t { prte_plm_base_component_t super; char *custom_args; bool early; bool ancient; int major; int minor; }; typedef struct prte_mca_plm_slurm_component_t prte_mca_plm_slurm_component_t; /* * Globally exported variable */ PRTE_MODULE_EXPORT extern prte_mca_plm_slurm_component_t prte_mca_plm_slurm_component; PRTE_EXPORT extern prte_plm_base_module_t prte_plm_slurm_module; END_C_DECLS #endif /* PRTE_PLM_SLURM_EXPORT_H */ prrte-4.1.1~rc2/src/mca/plm/slurm/configure.m40000664000175000017500000000305615203542132021347 0ustar alastairalastair# -*- shell-script -*- # # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2009-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2011-2016 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2017-2019 Intel, Inc. All rights reserved. # Copyright (c) 2022 Nanook Consulting. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # # MCA_plm_slurm_CONFIG([action-if-found], [action-if-not-found]) # ----------------------------------------------------------- AC_DEFUN([MCA_prte_plm_slurm_CONFIG],[ AC_CONFIG_FILES([src/mca/plm/slurm/Makefile]) PRTE_CHECK_SLURM([plm_slurm], [plm_slurm_good=1], [plm_slurm_good=0]) # if check worked, set wrapper flags if so. # Evaluate succeed / fail AS_IF([test "$plm_slurm_good" = "1"], [$1], [$2]) # there are no libraries to add to the Makefile as we don't # link against any Slurm libs ])dnl prrte-4.1.1~rc2/src/mca/plm/slurm/Makefile.am0000664000175000017500000000342115203542132021154 0ustar alastairalastair# # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2020 Cisco Systems, Inc. All rights reserved # Copyright (c) 2017 IBM Corporation. All rights reserved. # Copyright (c) 2017-2020 Intel, Inc. All rights reserved. # Copyright (c) 2022-2025 Nanook Consulting All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # EXTRA_DIST = \ help-plm-slurm.txt sources = \ plm_slurm.h \ plm_slurm_component.c \ plm_slurm_module.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). if MCA_BUILD_prte_plm_slurm_DSO component_noinst = component_install = prte_mca_plm_slurm.la else component_noinst = libprtemca_plm_slurm.la component_install = endif mcacomponentdir = $(prtelibdir) mcacomponent_LTLIBRARIES = $(component_install) prte_mca_plm_slurm_la_SOURCES = $(sources) prte_mca_plm_slurm_la_LDFLAGS = -module -avoid-version prte_mca_plm_slurm_la_LIBADD = $(top_builddir)/src/libprrte.la noinst_LTLIBRARIES = $(component_noinst) libprtemca_plm_slurm_la_SOURCES =$(sources) libprtemca_plm_slurm_la_LDFLAGS = -module -avoid-version prrte-4.1.1~rc2/src/mca/plm/base/0000775000175000017500000000000015203542132016670 5ustar alastairalastairprrte-4.1.1~rc2/src/mca/plm/base/plm_base_launch_support.c0000664000175000017500000026725115203542132023761 0ustar alastairalastair/* * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2020 Cisco Systems, Inc. All rights reserved * Copyright (c) 2009 Institut National de Recherche en Informatique * et Automatique. All rights reserved. * Copyright (c) 2011-2012 Los Alamos National Security, LLC. * Copyright (c) 2013-2020 Intel, Inc. All rights reserved. * Copyright (c) 2014-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2020 IBM Corporation. All rights reserved. * Copyright (c) 2021-2026 Nanook Consulting All rights reserved. * Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow * * $HEADER$ * */ #include "prte_config.h" #include "constants.h" #ifdef HAVE_SYS_WAIT_H # include #endif #ifdef HAVE_SYS_TIME_H # include #endif /* HAVE_SYS_TIME_H */ #include #include "src/class/pmix_pointer_array.h" #include "src/hwloc/hwloc-internal.h" #include "src/include/hash_string.h" #include "src/pmix/pmix-internal.h" #include "src/prted/pmix/pmix_server.h" #include "src/mca/errmgr/errmgr.h" #include "src/mca/ess/ess.h" #include "src/mca/filem/base/base.h" #include "src/mca/filem/filem.h" #include "src/mca/grpcomm/base/base.h" #include "src/mca/iof/base/base.h" #include "src/mca/odls/base/base.h" #include "src/mca/odls/odls_types.h" #include "src/mca/ras/base/base.h" #include "src/mca/rmaps/base/base.h" #include "src/mca/rmaps/rmaps.h" #include "src/rml/rml_contact.h" #include "src/rml/rml.h" #include "src/mca/state/base/base.h" #include "src/mca/state/state.h" #include "src/runtime/prte_globals.h" #include "src/runtime/prte_locks.h" #include "src/runtime/prte_quit.h" #include "src/runtime/runtime.h" #include "src/threads/pmix_threads.h" #include "src/util/dash_host/dash_host.h" #include "src/util/hostfile/hostfile.h" #include "src/util/pmix_argv.h" #include "src/util/name_fns.h" #include "src/util/pmix_net.h" #include "src/util/nidmap.h" #include "src/util/pmix_printf.h" #include "src/util/proc_info.h" #include "src/util/pmix_environ.h" #include "src/util/session_dir.h" #include "src/util/pmix_show_help.h" #include "src/mca/plm/base/base.h" #include "src/mca/plm/base/plm_private.h" void prte_plm_base_set_slots(prte_node_t *node) { if (0 == strncmp(prte_set_slots, "cores", strlen(prte_set_slots))) { if (NULL != node->topology && NULL != node->topology->topo) { node->slots = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_CORE); } } else if (0 == strncmp(prte_set_slots, "sockets", strlen(prte_set_slots))) { if (NULL != node->topology && NULL != node->topology->topo) { node->slots = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_SOCKET); if (0 == node->slots) { /* some systems don't report sockets - in this case, * use numanodes */ node->slots = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_NUMANODE); } } } else if (0 == strncmp(prte_set_slots, "numas", strlen(prte_set_slots))) { if (NULL != node->topology && NULL != node->topology->topo) { node->slots = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_NUMANODE); } } else if (0 == strncmp(prte_set_slots, "hwthreads", strlen(prte_set_slots))) { if (NULL != node->topology && NULL != node->topology->topo) { node->slots = prte_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PU); } } else { /* must be a number */ node->slots = strtol(prte_set_slots, NULL, 10); } /* mark the node as having its slots "given" */ PRTE_FLAG_SET(node, PRTE_NODE_FLAG_SLOTS_GIVEN); } void prte_plm_base_daemons_reported(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_topology_t *t; prte_node_t *node; int i; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* if we are not launching, then we just assume that all * daemons share our topology */ if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, 0); if (NULL == node || NULL == node->topology) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); caddy->jdata->state = PRTE_JOB_STATE_FAILED_TO_START; PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FAILED_TO_START); /* cleanup */ PMIX_RELEASE(caddy); return; } t = node->topology; for (i = 1; i < prte_node_pool->size; i++) { if (NULL == (node = (prte_node_t *) pmix_pointer_array_get_item(prte_node_pool, i))) { continue; } if (NULL == node->topology) { node->topology = t; node->available = prte_hwloc_base_filter_cpus(node->topology->topo); } node->state = PRTE_NODE_STATE_UP; } } /* if this is an unmanaged allocation, then set the default * slots on each node as directed or using default */ if (!prte_managed_allocation || prte_set_slots_override) { caddy->jdata->total_slots_alloc = 0; for (i = 0; i < caddy->jdata->session->nodes->size; i++) { node = (prte_node_t *) pmix_pointer_array_get_item(caddy->jdata->session->nodes, i); if (NULL == node) { continue; } if (!PRTE_FLAG_TEST(node, PRTE_NODE_FLAG_SLOTS_GIVEN)) { PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:setting slots for node %s by %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), node->name, prte_set_slots)); prte_plm_base_set_slots(node); } caddy->jdata->total_slots_alloc += node->slots; } } else { /* for managed allocations, the total slots allocated is fixed at time of allocation */ caddy->jdata->total_slots_alloc = prte_ras_base.total_slots_alloc; } if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DISPLAY_ALLOC, NULL, PMIX_BOOL)) { prte_ras_base_display_alloc(caddy->jdata); } /* progress the job */ caddy->jdata->state = PRTE_JOB_STATE_DAEMONS_REPORTED; PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_VM_READY); /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_allocation_complete(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_node_t *node; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* if we don't want to launch, then we at least want * to map so we can see where the procs would have * gone - so skip to the mapping state */ if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, 0); if (NULL == node) { // should never happen PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FAILED_TO_START); PMIX_RELEASE(caddy); return; } prte_rmaps_base.require_hwtcpus = !prte_hwloc_base_core_cpus(node->topology->topo); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); } else { /* move the state machine along */ caddy->jdata->state = PRTE_JOB_STATE_ALLOCATION_COMPLETE; PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_LAUNCH_DAEMONS); } /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_daemons_launched(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* do NOT increment the state - we wait for the * daemons to report that they have actually * started before moving to the right state */ /* cleanup */ PMIX_RELEASE(caddy); } static void files_ready(int status, void *cbdata) { prte_job_t *jdata = (prte_job_t *) cbdata; if (PRTE_SUCCESS != status) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); } else { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_MAP); } } void prte_plm_base_vm_ready(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_node_t *node; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* progress the job */ caddy->jdata->state = PRTE_JOB_STATE_VM_READY; /* check the first daemon's node for topology * limitations - or the HNP's node if we didn't * launch any daemons */ node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, 1); if (NULL == node) { node = (prte_node_t*)pmix_pointer_array_get_item(prte_node_pool, 0); } if (NULL != node && NULL != node->topology && NULL != node->topology->topo) { prte_rmaps_base.require_hwtcpus = !prte_hwloc_base_core_cpus(node->topology->topo); } /* position any required files */ if (PRTE_SUCCESS != prte_filem.preposition_files(caddy->jdata, files_ready, caddy->jdata)) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FILES_POSN_FAILED); } /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_mapping_complete(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* move the state machine along */ caddy->jdata->state = PRTE_JOB_STATE_MAP_COMPLETE; PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_SYSTEM_PREP); /* cleanup */ PMIX_RELEASE(caddy); } /* catch spawn timeout */ static void spawn_timeout_cb(int fd, short event, void *cbdata) { prte_job_t *jdata = (prte_job_t *) cbdata; prte_timer_t *timer = NULL; pmix_proc_t proc; int timeout, *tp; char *st; pmix_byte_object_t bo; PRTE_HIDE_UNUSED_PARAMS(fd, event); PMIX_ACQUIRE_OBJECT(jdata); /* Display a useful message to the user */ tp = &timeout; if (!prte_get_attribute(&jdata->attributes, PRTE_SPAWN_TIMEOUT, (void **) &tp, PMIX_INT)) { /* This shouldn't happen, but at least don't segv / display *something* if it does */ timeout = -1; } if (prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER)) { prte_event_evtimer_del(timer->ev); PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch deleting timeout for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); PMIX_RELEASE(timer); prte_remove_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT); } pmix_asprintf(&st, "--------------------------------------------------------------------------\n" "The user-provided time limit for job launch has been reached:\n\n" " Timeout: %d seconds\n\n" "The job will now be aborted. Please check your environment to\n" "identify the source of the delay and try again.\n" "--------------------------------------------------------------------------\n", timeout); bo.bytes = st; bo.size = strlen(st); PMIX_LOAD_PROCID(&proc, jdata->nspace, PMIX_RANK_WILDCARD); PMIx_server_IOF_deliver(&proc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); /* abort the job */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_FAILED_TO_START); jdata->exit_code = PRTE_ERR_TIMEOUT; if (!prte_persistent) { PRTE_UPDATE_EXIT_STATUS(PRTE_ERR_TIMEOUT); } } void prte_plm_base_stack_trace_recv(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { pmix_byte_object_t pbo; pmix_data_buffer_t blob; char *st, *st2; int32_t cnt; pmix_proc_t name; char *hostname, *nspace; pid_t pid; prte_job_t *jdata = NULL; prte_timer_t *timer; prte_proc_t proc; pmix_pointer_array_t parray; int rc; pmix_byte_object_t bo; PMIX_DATA_BUFFER_CONSTRUCT(&blob); PRTE_HIDE_UNUSED_PARAMS(status, tag, cbdata); pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s: stacktrace recvd from %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(sender)); /* unpack the stack_trace blob */ cnt = 1; rc = PMIx_Data_unpack(NULL, buffer, &nspace, &cnt, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&blob); return; } jdata = prte_get_job_data_object(nspace); if (NULL == jdata) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); free(nspace); return; } free(nspace); while (PMIX_SUCCESS == (rc = PMIx_Data_unpack(NULL, buffer, &pbo, &cnt, PMIX_BYTE_OBJECT))) { rc = PMIx_Data_load(&blob, &pbo); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&blob); goto DONE; } /* first piece is the name of the process */ cnt = 1; rc = PMIx_Data_unpack(NULL, &blob, &name, &cnt, PMIX_PROC); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&blob); goto DONE; } rc = PMIx_Data_unpack(NULL, &blob, &hostname, &cnt, PMIX_STRING); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&blob); goto DONE; } rc = PMIx_Data_unpack(NULL, &blob, &pid, &cnt, PMIX_PID); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&blob); goto DONE; } pmix_asprintf(&st, "STACK TRACE FOR PROC %s (%s, PID %lu)\n", PRTE_NAME_PRINT(&name), hostname, (unsigned long) pid); PMIx_Argv_append_nosize(&jdata->traces, st); free(hostname); free(st); /* unpack the stack_trace until complete */ cnt = 1; while (PRTE_SUCCESS == (rc = PMIx_Data_unpack(NULL, &blob, &st, &cnt, PMIX_STRING))) { pmix_asprintf(&st2, "\t%s", st); // has its own newline PMIx_Argv_append_nosize(&jdata->traces, st2); free(st); free(st2); cnt = 1; } if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { PMIX_ERROR_LOG(rc); } PMIX_DATA_BUFFER_DESTRUCT(&blob); cnt = 1; } if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { PMIX_ERROR_LOG(rc); } DONE: jdata->ntraces++; if (prte_process_info.num_daemons == jdata->ntraces) { timer = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_TRACE_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER) && NULL != timer) { prte_event_evtimer_del(timer->ev); /* timer is an prte_timer_t object */ PMIX_RELEASE(timer); prte_remove_attribute(&jdata->attributes, PRTE_JOB_TRACE_TIMEOUT_EVENT); } /* output the results - note that the output might need to go to a * tool instead of just to stderr, so we use the PMIx IOF deliver * function to ensure it gets where it needs to go */ PMIX_LOAD_PROCID(&name, jdata->nspace, PMIX_RANK_WILDCARD); for (cnt=0; NULL != jdata->traces[cnt]; cnt++) { bo.bytes = jdata->traces[cnt]; bo.size = strlen(jdata->traces[cnt]); PMIx_server_IOF_deliver(&name, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); } /* abort the job */ PMIX_CONSTRUCT(&parray, pmix_pointer_array_t); /* create an object */ PMIX_LOAD_PROCID(&proc.name, jdata->nspace, PMIX_RANK_WILDCARD); cnt = pmix_pointer_array_add(&parray, &proc); if (PRTE_SUCCESS != (rc = prte_plm.terminate_procs(&parray))) { PRTE_ERROR_LOG(rc); } PMIX_DESTRUCT(&parray); } } static void stack_trace_timeout(int sd, short args, void *cbdata) { prte_timer_t *timer; prte_job_t *jdata = (prte_job_t *) cbdata; prte_proc_t proc; pmix_pointer_array_t parray; int rc; PRTE_HIDE_UNUSED_PARAMS(sd, args); /* clear the timer */ timer = NULL; if (prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER) && NULL != timer) { prte_event_evtimer_del(timer->ev); /* timer is an prte_timer_t object */ PMIX_RELEASE(timer); prte_remove_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT_EVENT); } /* abort the job */ PMIX_CONSTRUCT(&parray, pmix_pointer_array_t); /* create an object */ PMIX_LOAD_PROCID(&proc.name, jdata->nspace, PMIX_RANK_WILDCARD); pmix_pointer_array_add(&parray, &proc); if (PRTE_SUCCESS != (rc = prte_plm.terminate_procs(&parray))) { PRTE_ERROR_LOG(rc); } PMIX_DESTRUCT(&parray); } static void dump_job(prte_job_t *jdata) { pmix_proc_t pc; prte_proc_t *proc; pmix_byte_object_t bo; char *st; int i; PMIX_LOAD_PROCID(&pc, jdata->nspace, PMIX_RANK_WILDCARD); pmix_asprintf(&st, "DATA FOR JOB: %s\n", PRTE_JOBID_PRINT(jdata->nspace)); bo.bytes = st; bo.size = strlen(st); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); pmix_asprintf(&st, "\tNum apps: %d\tNum procs: %d\tJobState: %s\tAbort: %s\n", (int) jdata->num_apps, (int) jdata->num_procs, prte_job_state_to_str(jdata->state), (PRTE_FLAG_TEST(jdata, PRTE_JOB_FLAG_ABORTED)) ? "True" : "False"); bo.bytes = st; bo.size = strlen(st); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); pmix_asprintf(&st, "\tNum launched: %ld\tNum reported: %ld\tNum terminated: %ld\n\n\tProcs:\n", (long) jdata->num_launched, (long) jdata->num_reported, (long) jdata->num_terminated); bo.bytes = st; bo.size = strlen(st); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); for (i = 0; i < jdata->procs->size; i++) { if (NULL != (proc = (prte_proc_t *) pmix_pointer_array_get_item(jdata->procs, i))) { pmix_asprintf(&st, "\t\tRank: %s\tNode: %s\tPID: %u\tState: %s\tExitCode %d\n", PRTE_VPID_PRINT(proc->name.rank), (NULL == proc->node) ? "UNKNOWN" : proc->node->name, (unsigned int) proc->pid, prte_proc_state_to_str(proc->state), proc->exit_code); bo.bytes = st; bo.size = strlen(st); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); } } st = "\n"; bo.bytes = st; bo.size = strlen(st); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); } static int get_traces(prte_job_t *jdata) { prte_daemon_cmd_flag_t command = PRTE_DAEMON_GET_STACK_TRACES; pmix_data_buffer_t buffer; pmix_byte_object_t bo; pmix_proc_t pc; pmix_status_t rc; PMIX_LOAD_PROCID(&pc, jdata->nspace, PMIX_RANK_WILDCARD); bo.bytes = "Waiting for stack traces (this may take a few moments)...\n"; bo.size = strlen(bo.bytes); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); /* setup the buffer */ PMIX_DATA_BUFFER_CONSTRUCT(&buffer); /* pack the command */ rc = PMIx_Data_pack(NULL, &buffer, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buffer); return PRTE_ERROR; } /* pack the jobid */ rc = PMIx_Data_pack(NULL, &buffer, &jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buffer); return PRTE_ERROR; } /* goes to all daemons */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, &buffer))) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_DESTRUCT(&buffer); return PRTE_ERROR; } PMIX_DATA_BUFFER_DESTRUCT(&buffer); return PRTE_SUCCESS; } static void job_timeout_cb(int fd, short event, void *cbdata) { prte_job_t *jdata = (prte_job_t *) cbdata; prte_timer_t *timer = NULL; prte_proc_t *prc; prte_job_t *child; pmix_proc_t pc; int rc, timeout, *tp, i; pmix_pointer_array_t parray; pmix_byte_object_t bo; char *st; PRTE_HIDE_UNUSED_PARAMS(fd, event); PMIX_ACQUIRE_OBJECT(jdata); /* Display a useful message to the user */ tp = &timeout; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_TIMEOUT, (void **) &tp, PMIX_INT)) { /* This shouldn't happen, but at least don't segv / display *something* if it does */ timeout = -1; } pmix_asprintf(&st, "--------------------------------------------------------------------------\n" "The user-provided time limit for job execution has been reached:\n\n" " Timeout: %d seconds\n\n" "The job will now be aborted. Please check your code and/or\n" "adjust/remove the job execution time limit (as specified by --timeout\n" "command line option or MPIEXEC_TIMEOUT environment variable).\n" "--------------------------------------------------------------------------\n", timeout); bo.bytes = st; bo.size = strlen(st); PMIX_LOAD_PROCID(&pc, jdata->nspace, PMIX_RANK_WILDCARD); PMIx_server_IOF_deliver(&pc, PMIX_FWD_STDERR_CHANNEL, &bo, NULL, 0, NULL, NULL); free(st); PRTE_UPDATE_EXIT_STATUS(PRTE_ERR_TIMEOUT); /* see if they want proc states reported */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_REPORT_STATE, NULL, PMIX_BOOL)) { /* output the results - note that the output might need to go to a * tool instead of just to stderr, so we use the PMIx IOF deliver * function to ensure it gets where it needs to go. */ dump_job(jdata); } /* Do this for all its child jobs, if any */ PMIX_LIST_FOREACH(child, &jdata->children, prte_job_t) { dump_job(child); } /* see if they want stacktraces */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_STACKTRACES, NULL, PMIX_BOOL)) { /* if they asked for stack_traces, attempt to get them, but timeout * if we cannot do so */ rc = get_traces(jdata); if (PRTE_SUCCESS != rc) { goto giveup; } // get traces for child jobs too PMIX_LIST_FOREACH(child, &jdata->children, prte_job_t) { rc = get_traces(child); if (PRTE_SUCCESS != rc) { goto giveup; } } /* we will terminate after we get the stack_traces, but set a timeout * just in case we never hear back from everyone */ if (prte_stack_trace_wait_timeout > 0) { timer = PMIX_NEW(prte_timer_t); prte_event_evtimer_set(prte_event_base, timer->ev, stack_trace_timeout, jdata); timer->tv.tv_sec = prte_stack_trace_wait_timeout; timer->tv.tv_usec = 0; prte_set_attribute(&jdata->attributes, PRTE_JOB_TRACE_TIMEOUT_EVENT, PRTE_ATTR_LOCAL, timer, PMIX_POINTER); PMIX_POST_OBJECT(timer); prte_event_evtimer_add(timer->ev, &timer->tv); } return; } giveup: /* abort the job */ PMIX_CONSTRUCT(&parray, pmix_pointer_array_t); pmix_pointer_array_init(&parray, PRTE_GLOBAL_ARRAY_BLOCK_SIZE, PRTE_GLOBAL_ARRAY_MAX_SIZE, PRTE_GLOBAL_ARRAY_BLOCK_SIZE); prc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&prc->name, jdata->nspace, PMIX_RANK_WILDCARD); pmix_pointer_array_add(&parray, prc); PMIX_LIST_FOREACH(child, &jdata->children, prte_job_t) { prc = PMIX_NEW(prte_proc_t); PMIX_LOAD_PROCID(&prc->name, child->nspace, PMIX_RANK_WILDCARD); pmix_pointer_array_add(&parray, prc); } if (PRTE_SUCCESS != (rc = prte_plm.terminate_procs(&parray))) { PRTE_ERROR_LOG(rc); } for (i=0; i < parray.size; i++) { prc = (prte_proc_t *) pmix_pointer_array_get_item(&parray, i); if (NULL == prc) { continue; } pmix_pointer_array_set_item(&parray, i, NULL); PMIX_RELEASE(prc); } PMIX_DESTRUCT(&parray); } void prte_plm_base_setup_job(int fd, short args, void *cbdata) { int rc; prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_timer_t *timer = NULL; int time, *tp; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:setup_job", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME))); if (PRTE_JOB_STATE_INIT != caddy->job_state) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } /* update job state */ caddy->jdata->state = caddy->job_state; /* start by getting a jobid */ if (PMIX_NSPACE_INVALID(caddy->jdata->nspace)) { if (PRTE_SUCCESS != (rc = prte_plm_base_create_jobid(caddy->jdata))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } } /* if the spawn operation has a timeout assigned to it, setup the timer for it */ tp = &time; if (prte_get_attribute(&caddy->jdata->attributes, PRTE_SPAWN_TIMEOUT, (void **) &tp, PMIX_INT)) { /* setup a timer to monitor execution time */ timer = PMIX_NEW(prte_timer_t); timer->payload = caddy->jdata; prte_event_evtimer_set(prte_event_base, timer->ev, spawn_timeout_cb, caddy->jdata); timer->tv.tv_sec = time; timer->tv.tv_usec = 0; prte_set_attribute(&caddy->jdata->attributes, PRTE_SPAWN_TIMEOUT_EVENT, PRTE_ATTR_LOCAL, timer, PMIX_POINTER); PMIX_POST_OBJECT(timer); prte_event_evtimer_add(timer->ev, &timer->tv); } /* if the overall job has a timeout assigned to it, setup the timer for it */ tp = &time; if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_TIMEOUT, (void **) &tp, PMIX_INT)) { /* setup a timer to monitor execution time */ timer = PMIX_NEW(prte_timer_t); timer->payload = caddy->jdata; prte_event_evtimer_set(prte_event_base, timer->ev, job_timeout_cb, caddy->jdata); timer->tv.tv_sec = time; timer->tv.tv_usec = 0; prte_set_attribute(&caddy->jdata->attributes, PRTE_JOB_TIMEOUT_EVENT, PRTE_ATTR_LOCAL, timer, PMIX_POINTER); PMIX_POST_OBJECT(timer); prte_event_evtimer_add(timer->ev, &timer->tv); } // if we are not going to launch this job, then ensure we output something - otherwise, // we will simply silently exit if (prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL) && !prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DISPLAY_MAP, NULL, PMIX_BOOL) && !prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, NULL, PMIX_BOOL) && !prte_get_attribute(&caddy->jdata->attributes, PRTE_JOB_REPORT_BINDINGS, NULL, PMIX_BOOL)) { // default to the devel map prte_set_attribute(&caddy->jdata->attributes, PRTE_JOB_DISPLAY_DEVEL_MAP, PRTE_ATTR_GLOBAL, NULL, PMIX_BOOL); } /* set the job state to the next position */ PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_INIT_COMPLETE); /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_setup_job_complete(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* nothing to do here but move along */ PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_ALLOCATE); PMIX_RELEASE(caddy); } void prte_plm_base_complete_setup(int fd, short args, void *cbdata) { prte_job_t *jdata; prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s complete_setup on job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(caddy->jdata->nspace)); /* bozo check */ if (PRTE_JOB_STATE_SYSTEM_PREP != caddy->job_state) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } /* update job state */ caddy->jdata->state = caddy->job_state; /* convenience */ jdata = caddy->jdata; /* set the job state to the next position */ PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_LAUNCH_APPS); /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_launch_apps(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; prte_daemon_cmd_flag_t command; int rc; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* convenience */ jdata = caddy->jdata; if (PRTE_JOB_STATE_LAUNCH_APPS != caddy->job_state) { PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } /* update job state */ caddy->jdata->state = caddy->job_state; PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch_apps for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); /* pack the appropriate add_local_procs command */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_FIXED_DVM, NULL, PMIX_BOOL)) { command = PRTE_DAEMON_DVM_ADD_PROCS; } else { command = PRTE_DAEMON_ADD_LOCAL_PROCS; } rc = PMIx_Data_pack(NULL, &jdata->launch_msg, &command, 1, PMIX_UINT8); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } /* get the local launcher's required data */ if (PRTE_SUCCESS != (rc = prte_odls.get_add_procs_data(&jdata->launch_msg, jdata->nspace))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); } PMIX_RELEASE(caddy); return; } void prte_plm_base_send_launch_msg(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; prte_job_t *jdata; int rc; PRTE_HIDE_UNUSED_PARAMS(fd, args); /* convenience */ jdata = caddy->jdata; PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:send launch msg for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); /* if we don't want to launch the apps, now is the time to leave */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DO_NOT_LAUNCH, NULL, PMIX_BOOL)) { /* go ahead and register the job */ rc = prte_pmix_server_register_nspace(jdata); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* if we are persistent, then we remain alive - otherwise, declare * all jobs complete and terminate */ if (prte_persistent) { PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_TERMINATED); } else { prte_never_launched = true; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_ALL_JOBS_COMPLETE); } PMIX_RELEASE(caddy); return; } /* goes to all daemons */ if (PRTE_SUCCESS != (rc = prte_grpcomm.xcast(PRTE_RML_TAG_DAEMON, &jdata->launch_msg))) { PRTE_ERROR_LOG(rc); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_NEVER_LAUNCHED); PMIX_RELEASE(caddy); return; } PMIX_DATA_BUFFER_DESTRUCT(&jdata->launch_msg); PMIX_DATA_BUFFER_CONSTRUCT(&jdata->launch_msg); /* track that we automatically are considered to have reported - used * only to report launch progress */ caddy->jdata->num_daemons_reported++; /* cleanup */ PMIX_RELEASE(caddy); } int prte_plm_base_spawn_response(int32_t status, prte_job_t *jdata) { int rc; pmix_data_buffer_t *answer; int room, *rmptr; pmix_info_t *iptr; size_t ninfo; time_t timestamp; pmix_proc_t *nptr; void *tinfo; int n; char *name; pmix_data_array_t darray; prte_app_context_t *app; /* if the requestor simply told us to terminate, they won't * be waiting for a response */ if (PMIX_NSPACE_INVALID(jdata->originator.nspace)) { return PRTE_SUCCESS; } /* if the response has already been sent, don't do it again */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_SPAWN_NOTIFIED, NULL, PMIX_BOOL)) { return PRTE_SUCCESS; } /* if the requestor was a tool, use PMIx to notify them of * launch complete as they won't be listening on PRRTE oob */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_DVM_JOB, NULL, PMIX_BOOL)) { /* dvm job => launch was requested by a TOOL, so we notify the launch proxy * and NOT the originator (as that would be us) */ nptr = NULL; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_LAUNCH_PROXY, (void **) &nptr, PMIX_PROC) || NULL == nptr) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } /* direct an event back to our controller */ timestamp = time(NULL); PMIX_INFO_LIST_START(tinfo); /* target this notification solely to that one tool */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_CUSTOM_RANGE, nptr, PMIX_PROC); PMIX_PROC_RELEASE(nptr); /* pass the nspace of the spawned job */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_NSPACE, jdata->nspace, PMIX_STRING); for (n=0; n < jdata->apps->size; n++) { app = (prte_app_context_t *) pmix_pointer_array_get_item(jdata->apps, n); if (NULL == app) { continue; } /* if pset name was assigned, pass it */ if (prte_get_attribute(&app->attributes, PRTE_APP_PSET_NAME, (void**) &name, PMIX_STRING)) { PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_PSET_NAME, name, PMIX_STRING); free(name); } /* pass the argv from each app */ name = PMIx_Argv_join(app->argv, ' '); PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_APP_ARGV, name, PMIX_STRING); free(name); } /* not to be delivered to a default event handler */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); /* provide the timestamp */ PMIX_INFO_LIST_ADD(rc, tinfo, PMIX_EVENT_TIMESTAMP, ×tamp, PMIX_TIME); /* protect against loops */ PMIX_INFO_LIST_ADD(rc, tinfo, "prte.notify.donotloop", NULL, PMIX_BOOL); PMIX_INFO_LIST_CONVERT(rc, tinfo, &darray); if (PMIX_ERR_EMPTY == rc) { iptr = NULL; ninfo = 0; } else if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PRTE_UPDATE_EXIT_STATUS(rc); PMIX_INFO_LIST_RELEASE(tinfo); PMIX_PROC_RELEASE(nptr); return rc; } else { iptr = (pmix_info_t *) darray.array; ninfo = darray.size; } PMIX_INFO_LIST_RELEASE(tinfo); PMIx_Notify_event(PMIX_LAUNCH_COMPLETE, &prte_process_info.myproc, PMIX_RANGE_CUSTOM, iptr, ninfo, NULL, NULL); PMIX_INFO_FREE(iptr, ninfo); } rmptr = &room; if (!prte_get_attribute(&jdata->attributes, PRTE_JOB_ROOM_NUM, (void **) &rmptr, PMIX_INT)) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); return PRTE_ERR_NOT_FOUND; } /* if the originator is me, then just do the notification */ if (PMIX_CHECK_PROCID(&jdata->originator, PRTE_PROC_MY_NAME)) { pmix_server_notify_spawn(jdata->nspace, room, status); return PRTE_SUCCESS; } /* prep the response to the spawn requestor */ PMIX_DATA_BUFFER_CREATE(answer); /* pack the status */ rc = PMIx_Data_pack(NULL, answer, &status, 1, PMIX_INT32); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return prte_pmix_convert_status(rc); } /* pack the jobid */ rc = PMIx_Data_pack(NULL, answer, &jdata->nspace, 1, PMIX_PROC_NSPACE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return prte_pmix_convert_status(rc); } /* pack the room number */ rc = PMIx_Data_pack(NULL, answer, &room, 1, PMIX_INT); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return prte_pmix_convert_status(rc); } PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch sending dyn release of job %s to %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), PRTE_NAME_PRINT(&jdata->originator))); PRTE_RML_SEND(rc, jdata->originator.rank, answer, PRTE_RML_TAG_LAUNCH_RESP); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); PMIX_DATA_BUFFER_RELEASE(answer); return rc; } return PRTE_SUCCESS; } void prte_plm_base_post_launch(int fd, short args, void *cbdata) { prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; int32_t rc, n; prte_job_t *jdata; prte_proc_t *proc; prte_app_context_t *app; prte_timer_t *timer; char *file = NULL; FILE *fp; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* convenience */ jdata = caddy->jdata; /* if a timer was defined, cancel it */ if (prte_get_attribute(&jdata->attributes, PRTE_SPAWN_TIMEOUT_EVENT, (void **) &timer, PMIX_POINTER)) { prte_event_evtimer_del(timer->ev); PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch deleting spawn timeout for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); PMIX_RELEASE(timer); prte_remove_attribute(&jdata->attributes, PRTE_SPAWN_TIMEOUT_EVENT); } if (PRTE_JOB_STATE_RUNNING != caddy->job_state) { /* error mgr handles this */ PMIX_RELEASE(caddy); return; } /* update job state */ caddy->jdata->state = caddy->job_state; /* complete wiring up the iof */ PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch wiring up iof for job %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); /* if requested, output the proctable */ if (prte_get_attribute(&jdata->attributes, PRTE_JOB_OUTPUT_PROCTABLE, (void**)&file, PMIX_STRING)) { /* if file="-", then output to stdout */ if (0 == strcmp(file, "-")) { fp = stdout; } else if (0 == strcmp(file, "+")) { fp = stderr; } else { /* attempt to open the specified file */ fp = fopen(file, "w"); if (NULL == fp) { pmix_output(0, "Unable to open file %s for output of proctable", file); goto next; } } for (n=0; n < jdata->procs->size; n++) { proc = (prte_proc_t*)pmix_pointer_array_get_item(jdata->procs, n); if (NULL == proc) { continue; } app = (prte_app_context_t*)pmix_pointer_array_get_item(jdata->apps, proc->app_idx); if (NULL == app) { // should never happen continue; } fprintf(fp, "(rank, host, exe, pid) = (%u, %s, %s, %d)\n", proc->name.rank, proc->node->name, app->app, proc->pid); } if (stdout != fp && stderr != fp) { fclose(fp); } } next: /* notify the spawn requestor */ rc = prte_plm_base_spawn_response(PRTE_SUCCESS, jdata); if (PRTE_SUCCESS != rc) { PRTE_ERROR_LOG(rc); } /* cleanup */ PMIX_RELEASE(caddy); } void prte_plm_base_registered(int fd, short args, void *cbdata) { prte_job_t *jdata; prte_state_caddy_t *caddy = (prte_state_caddy_t *) cbdata; PRTE_HIDE_UNUSED_PARAMS(fd, args); PMIX_ACQUIRE_OBJECT(caddy); /* convenience */ jdata = caddy->jdata; PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch %s registered", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace))); if (PRTE_JOB_STATE_REGISTERED != caddy->job_state) { PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:launch job %s not registered - state %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_JOBID_PRINT(jdata->nspace), prte_job_state_to_str(caddy->job_state))); PRTE_ACTIVATE_JOB_STATE(caddy->jdata, PRTE_JOB_STATE_FORCED_EXIT); PMIX_RELEASE(caddy); return; } /* update job state */ jdata->state = caddy->job_state; PMIX_RELEASE(caddy); } /* daemons callback when they start - need to listen for them */ static void progress_daemons(prte_job_t *daemons, bool show_progress) { int i; prte_job_t *jdata; prte_proc_t *daemon; prte_topology_t *t; pmix_rank_t j; if (show_progress && (0 == daemons->num_reported % 100 || daemons->num_reported == prte_process_info.num_daemons)) { PRTE_ACTIVATE_JOB_STATE(daemons, PRTE_JOB_STATE_REPORT_PROGRESS); } PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:progress_daemons recvd %d of %d reported daemons", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), daemons->num_reported, daemons->num_procs)); if (daemons->num_procs == daemons->num_reported) { bool oneactivated = false; daemons->state = PRTE_JOB_STATE_DAEMONS_REPORTED; /* activate the daemons_reported state for all jobs * whose daemons were launched */ for (i = 1; i < prte_job_data->size; i++) { jdata = (prte_job_t *) pmix_pointer_array_get_item(prte_job_data, i); if (NULL == jdata) { continue; } if (PRTE_JOB_STATE_DAEMONS_LAUNCHED == jdata->state) { oneactivated = true; PRTE_ACTIVATE_JOB_STATE(jdata, PRTE_JOB_STATE_DAEMONS_REPORTED); } } if (!oneactivated) { /* must be launching a DVM - activate the state */ PRTE_ACTIVATE_JOB_STATE(daemons, PRTE_JOB_STATE_DAEMONS_REPORTED); } if (prte_homo_nodes && 1 < daemons->num_procs) { // ensure that all topologies point to the one // returned by daemon rank=1 as it might be different // from where prterun is executing (e.g., login node) daemon = (prte_proc_t *) pmix_pointer_array_get_item(daemons->procs, 1); if (NULL == daemon->node->topology) { // should never happen } t = daemon->node->topology; for (j=2; j < daemons->num_procs; j++) { daemon = (prte_proc_t *) pmix_pointer_array_get_item(daemons->procs, j); if (NULL == daemon) { continue; } daemon->node->topology = t; /* update the node's available processors */ if (NULL != daemon->node->available) { hwloc_bitmap_free(daemon->node->available); } daemon->node->available = prte_hwloc_base_filter_cpus(t->topo); } } } } void prte_plm_base_daemon_callback(int status, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { char *ptr; int idx; pmix_status_t ret; prte_proc_t *daemon = NULL; pmix_proc_t dname; prte_topology_t *t; int i; bool found, show_progress; char *alias; char *nodename = NULL; pmix_byte_object_t pbo, bo; bool compressed; pmix_data_buffer_t datbuf, *data; pmix_topology_t ptopo; pmix_value_t cnctinfo; hwloc_topology_diff_t diff; bool prted_failed_launch = false; prte_job_t *jdatorted = NULL; PRTE_HIDE_UNUSED_PARAMS(status, sender, tag, cbdata); /* get the daemon job */ jdatorted = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); show_progress = prte_get_attribute(&jdatorted->attributes, PRTE_JOB_SHOW_PROGRESS, NULL, PMIX_BOOL); /* multiple daemons could be in this buffer, so unpack until we exhaust the data */ idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &dname, &idx, PMIX_PROC); while (PMIX_SUCCESS == ret) { pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s plm:base:prted_report_launch from daemon %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&dname)); /* update state and record for this daemon contact info */ daemon = (prte_proc_t *) pmix_pointer_array_get_item(jdatorted->procs, dname.rank); if (NULL == daemon) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); prted_failed_launch = true; goto CLEANUP; } daemon->state = PRTE_PROC_STATE_RUNNING; /* record that this daemon is alive */ PRTE_FLAG_SET(daemon, PRTE_PROC_FLAG_ALIVE); /* unload its contact info */ PMIX_VALUE_CONSTRUCT(&cnctinfo); cnctinfo.type = PMIX_STRING; idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &cnctinfo.data.string, &idx, PMIX_STRING); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } /* store this for later distribution */ ret = PMIx_Store_internal(&dname, PMIX_PROC_URI, &cnctinfo); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); PMIX_VALUE_DESTRUCT(&cnctinfo); prted_failed_launch = true; goto CLEANUP; } daemon->rml_uri = strdup(cnctinfo.data.string); PMIX_VALUE_DESTRUCT(&cnctinfo); /* unpack the node name */ idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &nodename, &idx, PMIX_STRING); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } if (!pmix_net_isaddr(nodename) && NULL != (ptr = strchr(nodename, '.'))) { /* retain the non-fqdn name as an alias */ *ptr = '\0'; PMIx_Argv_append_unique_nosize(&daemon->node->aliases, nodename); *ptr = '.'; } PMIX_OUTPUT_VERBOSE((5, prte_plm_base_framework.framework_output, "%s plm:base:prted_report_launch from daemon %s on node %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), PRTE_NAME_PRINT(&daemon->name), nodename)); /* mark the daemon as launched */ PRTE_FLAG_SET(daemon->node, PRTE_NODE_FLAG_DAEMON_LAUNCHED); daemon->node->state = PRTE_NODE_STATE_UP; /* first, store the nodename itself. in case the nodename isn't * the same as what we were given by the allocation, we replace * the node's name with the returned value and store the allocation * value as an alias. For example, a hostfile * might contain an IP address instead of the value returned * by gethostname, yet the daemon will have returned the latter * and apps may refer to the host by that name */ if (0 != strcmp(nodename, daemon->node->name)) { PMIx_Argv_append_unique_nosize(&daemon->node->aliases, daemon->node->name); free(daemon->node->name); daemon->node->name = strdup(nodename); } /* unpack and store the provided aliases */ idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &alias, &idx, PMIX_STRING); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } if (NULL != alias) { daemon->node->aliases = PMIx_Argv_split(alias, ','); free(alias); } if (0 < pmix_output_get_verbosity(prte_plm_base_framework.framework_output)) { int ni; pmix_output(0, "ALIASES FOR NODE %s (%s)", daemon->node->name, nodename); if (NULL != daemon->node->aliases) { for (ni=0; NULL != daemon->node->aliases[ni]; ni++) { pmix_output(0, "\tALIAS: %s", daemon->node->aliases[ni]); } } } if (!prte_homo_nodes || 1 == daemon->name.rank) { /* unpack the topology for that node */ PMIX_DATA_BUFFER_CONSTRUCT(&datbuf); /* unpack the flag to see if this payload is compressed */ idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &compressed, &idx, PMIX_BOOL); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } /* unpack the data */ idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &pbo, &idx, PMIX_BYTE_OBJECT); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } if (compressed) { /* decompress the data */ if (PMIx_Data_decompress((uint8_t *) pbo.bytes, pbo.size, (uint8_t **) &bo.bytes, &bo.size)) { /* the data has been uncompressed */ ret = PMIx_Data_load(&datbuf, &bo); PMIX_BYTE_OBJECT_DESTRUCT(&bo); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; PMIX_BYTE_OBJECT_DESTRUCT(&pbo); goto CLEANUP; } } else { pmix_show_help("help-prte-runtime.txt", "failed-to-uncompress", true, prte_process_info.nodename); prted_failed_launch = true; PMIX_BYTE_OBJECT_DESTRUCT(&pbo); PMIX_BYTE_OBJECT_DESTRUCT(&bo); goto CLEANUP; } } else { ret = PMIx_Data_load(&datbuf, &pbo); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; PMIX_BYTE_OBJECT_DESTRUCT(&pbo); goto CLEANUP; } } PMIX_BYTE_OBJECT_DESTRUCT(&pbo); data = &datbuf; /* unpack the topology information */ idx = 1; ret = PMIx_Data_unpack(NULL, data, &ptopo, &idx, PMIX_TOPO); if (PMIX_SUCCESS != ret) { PMIX_ERROR_LOG(ret); prted_failed_launch = true; goto CLEANUP; } /* cleanup */ PMIX_DATA_BUFFER_DESTRUCT(data); pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s RECEIVED TOPOLOGY FROM NODE %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), nodename); /* check to see if we already have this topology from some other node, * and if we do, record the diff for this node */ found = false; for (i = 0; i < prte_node_topologies->size; i++) { t = (prte_topology_t *) pmix_pointer_array_get_item(prte_node_topologies, i); if (NULL == t) { continue; } /* compute the diff */ ret = hwloc_topology_diff_build(t->topo, ptopo.topology, 0, &diff); if (0 == ret) { pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s TOPOLOGY ALREADY RECORDED IN POSN %d - %s DIFFS FOUND", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), i, (NULL == diff) ? "NO" : "SOME"); daemon->node->topology = t; found = true; /* update the node's available processors */ if (NULL != daemon->node->available) { hwloc_bitmap_free(daemon->node->available); } daemon->node->available = prte_hwloc_base_filter_cpus(t->topo); daemon->node->topodiff = diff; // release the unpacked topology PMIX_TOPOLOGY_DESTRUCT(&ptopo); break; } } if (!found) { // this is a new topology t = PMIX_NEW(prte_topology_t); t->topo = ptopo.topology; // we don't use the source field if (NULL != ptopo.source) { free(ptopo.source); } t->index = pmix_pointer_array_add(prte_node_topologies, t); pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s ADDING NEW TOPOLOGY AT POSN %d", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), t->index); daemon->node->topology = t; daemon->node->available = prte_hwloc_base_filter_cpus(t->topo); prte_hwloc_base_setup_summary(t->topo); } } // mark as completed jdatorted->num_reported++; jdatorted->num_daemons_reported++; CLEANUP: pmix_output_verbose(5, prte_plm_base_framework.framework_output, "%s plm:base:prted_daemon_cback %s for daemon %s at contact %s", PRTE_NAME_PRINT(PRTE_PROC_MY_NAME), prted_failed_launch ? "failed" : "completed", PRTE_NAME_PRINT(&dname), (NULL == daemon) ? "UNKNOWN" : daemon->rml_uri); if (NULL != nodename) { free(nodename); nodename = NULL; } idx = 1; ret = PMIx_Data_unpack(NULL, buffer, &dname, &idx, PMIX_PROC); if (PMIX_SUCCESS != ret) { break; } } if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != ret) { PMIX_ERROR_LOG(ret); PRTE_ACTIVATE_JOB_STATE(jdatorted, PRTE_JOB_STATE_FAILED_TO_START); return; } if (prted_failed_launch) { PRTE_ACTIVATE_JOB_STATE(jdatorted, PRTE_JOB_STATE_FAILED_TO_START); return; } progress_daemons(jdatorted, show_progress); } void prte_plm_base_daemon_failed(int st, pmix_proc_t *sender, pmix_data_buffer_t *buffer, prte_rml_tag_t tag, void *cbdata) { int status, rc; int32_t n; pmix_rank_t vpid; prte_proc_t *daemon = NULL; prte_job_t *jdatorted; PRTE_HIDE_UNUSED_PARAMS(st, sender, tag, cbdata); /* get the daemon job */ jdatorted = prte_get_job_data_object(PRTE_PROC_MY_NAME->nspace); /* unpack the daemon that failed */ n = 1; rc = PMIx_Data_unpack(NULL, buffer, &vpid, &n, PMIX_PROC_RANK); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); goto finish; } /* unpack the exit status */ n = 1; rc = PMIx_Data_unpack(NULL, buffer, &status, &n, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); status = PRTE_ERROR_DEFAULT_EXIT_CODE; PRTE_UPDATE_EXIT_STATUS(PRTE_ERROR_DEFAULT_EXIT_CODE); } else { PRTE_UPDATE_EXIT_STATUS(WEXITSTATUS(status)); } /* find the daemon and update its state/status */ if (NULL == (daemon = (prte_proc_t *) pmix_pointer_array_get_item(jdatorted->procs, vpid))) { PRTE_ERROR_LOG(PRTE_ERR_NOT_FOUND); goto finish; } daemon->state = PRTE_PROC_STATE_FAILED_TO_START; daemon->exit_code = status; finish: if (NULL == daemon) { PRTE_ACTIVATE_JOB_STATE(NULL, PRTE_PROC_STATE_FAILED_TO_START); return; } else { PRTE_ACTIVATE_PROC_STATE(&daemon->name, PRTE_PROC_STATE_FAILED_TO_START); } } int prte_plm_base_setup_prted_cmd(int *argc, char ***argv) { int i, loc; char **tmpv; /* set default location to be 0, indicating that * only a single word is in the cmd */ loc = 0; /* split the command apart in case it is multi-word */ tmpv = PMIx_Argv_split(prte_launch_agent, ' '); for (i = 0; NULL != tmpv && NULL != tmpv[i]; ++i) { if (0 == strcmp(tmpv[i], "prted")) { loc = i; } pmix_argv_append(argc, argv, tmpv[i]); } PMIx_Argv_free(tmpv); return loc; } /* pass all options as MCA params so anything we pickup * from the environment can be checked for duplicates */ int prte_plm_base_prted_append_basic_args(int *argc, char ***argv, char *ess, int *proc_vpid_index) { char *param = NULL, **tmpv; int i, j, cnt, offset; prte_job_t *jdata; unsigned long num_procs; bool ignore; char *skips[] = { "rmaps", "ras", NULL }; /* check for debug flags */ if (prte_debug_flag) { pmix_argv_append(argc, argv, "--debug"); } if (prte_debug_daemons_flag) { pmix_argv_append(argc, argv, "--debug-daemons"); } if (prte_debug_daemons_file_flag) { pmix_argv_append(argc, argv, "--debug-daemons-file"); } if (prte_leave_session_attached) { pmix_argv_append(argc, argv, "--leave-session-attached"); } if (prte_allow_run_as_root) { pmix_argv_append(argc, argv, "--allow-run-as-root"); } if (prte_homo_nodes) { pmix_argv_append(argc, argv, "--uniform-nodes"); } /* the following is not an mca param */ if (NULL != getenv("PRTE_TEST_PRTED_SUICIDE")) { pmix_argv_append(argc, argv, "--test-suicide"); } /* tell the orted what ESS component to use */ if (NULL != ess) { pmix_argv_append(argc, argv, "--prtemca"); pmix_argv_append(argc, argv, "ess"); pmix_argv_append(argc, argv, ess); } /* pass the daemon nspace */ pmix_argv_append(argc, argv, "--prtemca"); pmix_argv_append(argc, argv, "ess_base_nspace"); pmix_argv_append(argc, argv, prte_process_info.myproc.nspace); free(param); /* setup to pass the vpid */ if (NULL != proc_vpid_index) { pmix_argv_append(argc, argv, "--prtemca"); pmix_argv_append(argc, argv, "ess_base_vpid"); *proc_vpid_index = *argc; pmix_argv_append(argc, argv, "