#!/bin/sh
###############################################################################
# WebSphere Application Server liberty launch script
#
# Copyright (c) 2011, 2023 IBM Corporation and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License 2.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-2.0/
# 
# SPDX-License-Identifier: EPL-2.0
# 
# ----------------------------------------------------------------------------
#
# To customize the use of this script (for example with /etc/init.d system 
# service managers), use the following environment variables:
#
# JAVA_HOME  - The java executable is found in ${JAVA_HOME}/bin
#
# JVM_ARGS   - A list of JVM command line options,
#              e.g. system properties or -X parameters
#              The value will be expanded by the shell (use quotes for spaces)
#
# LOG_DIR    - The log file directory
#              The default value is ${WLP_OUTPUT_DIR}/serverName/logs
#
# LOG_FILE   - The log file name
#              This log file is only used if the server is run in the
#              background via the start action. 
#              The default value is console.log
#
# CRIU_LOG_LEVEL - Set the log level of criu checkpoint and restore operations.
#                  Set to 1-4 with 4 being max verbosity. Two "2" is the default.
#
# CRIU_UNPRIVILEGED - If true, True, TRUE, or 1 the --unprivileged option is 
#                     passed to criu during checkpoint operations. If set to any other 
#                     value the switch will not be passed in. 
#                     If unset, the unprivileged switch is passed in unless the current user is root
#
# CRIU_RESTORE_DISABLE_RECOVERY - 
#              When restoring a server from a checkpoint it is possible
#              that CRIU will fail to restore the process.  By default
#              any failure to restore will result in the server
#              recovering without using the checkpoint image.
#              To disable this behavior set this to true.
#
# PID_DIR    - The directory that should be used for server pid file(s).
#              The default value is ${WLP_OUTPUT_DIR}/.pid
#
# PID_FILE   - The file that should be used for the server pid file.
#              The default value is ${PID_DIR}/serverName.pid
#
# WLP_USER_DIR - The user/custom configuration directory used to store
#              shared and server-specific configuration. 
#              See README.TXT for details about shared resource locations.
#              A server's configuration is at ${WLP_USER_DIR}/servers/serverName
#              The default value is the usr directory in the install directory.
#
# WLP_OUTPUT_DIR - The directory containing output files for defined servers.
#              This directory must have both read and write permissions for
#              the user or users that start servers.
#              By default, a server's output logs and workarea are stored
#              in the ${WLP_USER_DIR}/servers/serverName directory
#              (alongside configuration and applications).
#              If this variable is set, the output logs and workarea 
#              would be stored in ${WLP_OUTPUT_DIR}/serverName.
#
# WLP_DEBUG_ADDRESS - The port to use when running the server in debug mode.
#              The default value is 7777.
#
# WLP_DEBUG_SUSPEND - Whether to suspend the jvm on startup or not. This can be
#              set to y to suspend the jvm on startup until a debugger attaches,
#              or set to n to startup without waiting for a debugger to attach.
#              The default value is y. 
#
# WLP_DEBUG_REMOTE - Whether to allow remote debugging or not. This can be set
#              to y to allow remote debugging. By default, this value is not
#              defined, which does not allow remote debugging on newer JDK/JREs.
#
# WLP_SKIP_UMASK -  Skip setting the umask value to allow the default value 
#              to be used.
#
# WLP_ZOS_PROCEDURE - The name of the STC procedure to be used to start the 
#              server in a z/OS environment.
#
# WLP_ZOS_JOBNAME - The job name to use when starting the server in
#              a z/OS environment.
#
# SERVER_WORKING_DIR - The directory containing output files from the JVM.  For 
#              example the javadump files.  The default value is the location
#              ${WLP_OUTPUT_DIR}/serverName. If set with an absolute path 
#              (ex, /logs), that location will be utilized, else if a relative 
#              path (ex, logs) is set it will be relative to the default 
#              location.
# WLP_IMMUTABLE_VARS - The comma separated list of environment variable names
#              that will have immutable values between when a server checkpoint
#              action is done and when the server is restored.
#
###############################################################################
SERVER_UNKNOWN_STATUS=5

##
## Determine the platform and absolute path of the installation directory.
##
case $OSTYPE in
  cygwin)
    uname=CYGWIN_NT

    # Determine the installation directory without forking if possible.  Use
    # eval to hide ${var//find/replace}, ${var%suffix}, and ${var:first:length}
    # syntaxes from shells that can't parse them.
    eval '
      # cd to the install directory.
      savePWD=$PWD
      script=${0//\\/\/}
      unset CDPATH; cd "${script%/*}"/..

      # Convert the install (current working) directory to a Windows path.
      case $PWD in
        /cygdrive/?/*)
          # Use ${var:first:length} to avoid forking for cygpath.
          WLP_INSTALL_DIR=${PWD:10:1}:${PWD:11}
          ;;
        *)
          WLP_INSTALL_DIR=`cygpath -ma .`
      esac

      cd "$savePWD"
    '
    ;;

  *)
    uname=`uname`

    case $uname in
      CYGWIN_*)
        WLP_INSTALL_DIR=`cygpath -ma "${0}"/../..`
        ;;

      *)
        dirname=`dirname "$0"`
        WLP_INSTALL_DIR=`unset CDPATH; cd "$dirname/.." && pwd`
    esac
esac

##
## Platform specific setup
##
PS_P='ps -p'
UMASK_O='umask o='
tryShellExtensions=true
os400lib=false
shareclassesCacheDirPerm=true
defaultFileEncoding=
newline='
'
nativeEBCDIC=false

case ${uname} in
  CYGWIN_*)
    # java.exe is a non-Cygwin process, so we need to pass -W.
    PS_P='ps -W -p'
    shareclassesCacheDirPerm=false
    ;;

  OS/390)
    defaultFileEncoding=iso8859-1
    nativeEBCDIC=true         # Auto-convert server.env/jvm.options from ASCII-to-EBCDIC, if necessary.
    _BPXK_WLM_PROPAGATE=NO    # Prevent WLM enclave propagation to spawned processes
    _EDC_PTHREAD_YIELD=-2     # Disable sleeps inside of pthread_yield
    JAVA_PROPAGATE=NO         # Prevent WLM enclave propagation to new threads
    export _BPXK_WLM_PROPAGATE _EDC_PTHREAD_YIELD JAVA_PROPAGATE

    WLP_NLS_PATH="${WLP_INSTALL_DIR}/lib/native/zos/s390x/nls/%N.cat"
    if [ ${NLSPATH} ]
    then
      NLSPATH="${WLP_NLS_PATH}:${NLSPATH}"
    else
      NLSPATH="${WLP_NLS_PATH}"
    fi
    ;;

  OS400)
    tryShellExtensions=false
    # os400lib gets unset later if we cannot find WAS_PRODUCT_LIB and WAS_SHARED_LIB
    os400lib=true
    # on os400 determine current umask and replace o value with 7 (deny all permissions for other)
    UMASK_O="umask `/QOpenSys/usr/bin/umask | sed 's|[0-7]$|7|g'`"
    ;;
esac

##
## safeEcho: Portable echo that can handle backslashes and leading dashes.
safeEcho()
{
  cat <<EOF
$*
EOF

  return $?
}

# Starting the server with a full tmp directory may prevent the server from
# reading server.env during startup.
safeEcho "" || exit $?

# Define escapeForEval functions using ${var//find/replace} and ${var#suffix}
# if possible since those constructs are significantly faster than safeEcho+sed
# since they avoid forks.  Use eval (to hide the syntax from shells that don't
# support them) in a subshell (to avoid exiting the shell process on error) to
# test if the shell has support.
if ${tryShellExtensions} && (eval 'true ${1//b/c} ${1#*=}') 2> /dev/null; then
  # The shell has support.  Define the functions using eval, again to hide the
  # syntax from shells that don't support it.
  eval "
    escapeForEval()
    {
      escapeForEvalResult=\\'\${1//\\'/\\'\\\"\\'\\\"\\'}\\'
    }

    extractValueAndEscapeForEval()
    {
      escapeForEval \"\${1#*=}\"
    }

    substitutePrefixVar()
    {
      case \$1 in
      @\$2@*) substitutePrefixVarResult=\$3\${1#@\$2@};;
      *) substitutePrefixVarResult=\$1
      esac
    }
  "
else
  ##
  ## escapeForEval: Escape the first parameter to be safe for use in eval,
  ##                and set escapeForEvalResult with the result.
  ##
  escapeForEval()
  {
    escapeForEvalResult=\'`safeEcho "$1" | sed s/\'/\'\"\'\"\'/g`\'
  }

  ##
  ## extractValueAndEscapeForEval: Extract the value of a var=value string,
  ##                               and set escapeForEvalResult with the result.
  ##
  extractValueAndEscapeForEval()
  {
    escapeForEvalResult=\'`safeEcho "$1" | sed -e 's/[^=]*=//' -e s/\'/\'\"\'\"\'/g`\'
  }

  ##
  ## substitutePrefixVar: If $1 has a prefix @$2@, set substitutePrefixVarResult
  ##                      to $1 with the prefix replaced by $3.  Otherwise, set
  ##                      to $1.
  substitutePrefixVar()
  {
    case $1 in
    @$2@*) substitutePrefixVarResult=$3`safeEcho "$1" | sed -e "s/^@$2@//"`;;
    *) substitutePrefixVarResult=$1
    esac
  }
fi

##
## Quote ${WLP_INSTALL_DIR} for eval.
##
escapeForEval "${WLP_INSTALL_DIR}"
WLP_INSTALL_DIR_QUOTED=${escapeForEvalResult}
READ_ETC=1

##
## Consume script parameters:
##   action is required/positional,
##   serverName is optional, --options following
##
if [ $# -lt 1 ]
then
  ACTION=help:usage
else
  ACTION=$1
  shift #consume
  if [ $# -ge 1 ]
  then
    # Only use if it isn't something that looks like an option
    case $1 in
    --*);;
    *)
      SERVER_NAME=$1
      shift #consume
    esac
  fi
fi

##
## Set server name and directory
##
if [ -z "$SERVER_NAME" ]
then
  SERVER_NAME=defaultServer
fi

##
## Set JAVA_AGENT_QUOTED if WLP_SKIP_BOOTSTRAP_AGENT is unset.
##
JAVA_AGENT_QUOTED=-javaagent:${WLP_INSTALL_DIR_QUOTED}/bin/tools/ws-javaagent.jar
if [ -n "${WLP_SKIP_BOOTSTRAP_AGENT}" ]; then
  JAVA_AGENT_QUOTED=
fi

##
## Determine which CRIU mode to use, based on current euid, and the value of the CRIU_UNPRIVILEGED env var.
## Set DO_CRIU_UNPRIVILEGED based on the above.
##
checkCriuUnprivileged()
{
  # Use the environment variable, if it's set, to determine mode.
  if [ -n "$CRIU_UNPRIVILEGED" ]; then
    if [ "${CRIU_UNPRIVILEGED}" = "true" -o "${CRIU_UNPRIVILEGED}" = "1" -o "${CRIU_UNPRIVILEGED}" = "TRUE" -o "${CRIU_UNPRIVILEGED}" = "True" ]; then
      DO_CRIU_UNPRIVILEGED=true
    else
      DO_CRIU_UNPRIVILEGED=false
    fi
    return
  fi

  # Environment variable CRIU_UNPRIVILEGED is not set so determine which CRIU mode to use based on current euid. 
  if [ "$(id -u)" = 0 ]; then
    # root user
    DO_CRIU_UNPRIVILEGED=false
  else  
    # Not root, use unprivileged mode in criu
    DO_CRIU_UNPRIVILEGED=true
  fi
}

##
## createServer: Function to launch server create
##
createServer()
{
  javaCmd '' "${SERVER_NAME}" --create "$@"
  rc=$?

  if [ $rc = 0 ]; then
    rmIfExist "${X_PID_FILE}"

    if $os400lib; then
      # If production on IBM i, authorize QEJBSVR to the server folder
      chgautOS400 "${WLP_USER_DIR}/servers/${SERVER_NAME}" "*RX"
      chgautOS400 "${WLP_OUTPUT_DIR}" "*RWX"
      if [ -d "${WLP_USER_DIR}/servers/${SERVER_NAME}/apps" ]; then
         chgautOS400 "${WLP_USER_DIR}/servers/${SERVER_NAME}/apps" "*RWX"
      fi

    fi
  fi

  return $rc
}

##
## chgautOS400: Change authorization of directory or file $1 to $2 for QEJBSVR.
##              This should only be called if $os400lib is true
##
chgautOS400()
{
  system "CHGAUT OBJ('$1') USER(QEJBSVR) DTAAUT($2) OBJAUT(*ALL) SUBTREE(*ALL)"
}

##
## mkdirs: Create a directory and all parent directories, if necessary.
##
mkdirs()
{
  if [ ! -d "$1" ]; then
    mkdir -p "$1"
    if $os400lib; then
      chgautOS400 "$1" "*RWX"
    fi
  fi
}

backupIfNeededOrRemove()
{
  if [ -f "$2" ]; then
    # backup already exists remove file
    rm -f "$1"
  fi
  if [ -f "$1" ]; then
    mv "$1" "$2"
  fi
}

##
## rmIfExist: Remove files if they exist.
##
rmIfExist()
{
  for file
  do
    # [ is a builtin command for most shells, so a fork is not required.
    if [ -f "${file}" ]; then
      # If any file exists, pass all files so we only fork once.
      rm -f "$@"
      return
    fi
  done
}

##
## touchIfNotExist: Create an empty file.
##
touchIfNotExist()
{
  if [ ! -f "$1" ]; then
    # true is a builtin command for most shells, so a fork is not required.
    true > "$1"
    if $os400lib; then
      chgautOS400 "$1" "*RWX"
    fi
  fi
}

##
## serverStatus: Check for a running server process.  If the first argument is
##               true, avoid using the launcher to verify the status if possible,
##               and don't output messages even if the launcher is used.
##               The second argument is the --status variant.
##               If success is returned, then ${PID} will be set if available.
serverStatus()
{
  if [ -f "${SERVER_OUTPUT_DIR}/workarea/.sLock" ]; then
    if [ ! -f "${SERVER_OUTPUT_DIR}/workarea/.sCommand" ]; then
      # Workarea exists, .sCommand doesn't.  Server is not running.
      rmIfExist "${X_PID_FILE}"
      return 1
    fi

    # Read the PID file to set ${PID} if possible.  Remove it if stale.
    # Otherwise, assume the server is started if we're avoiding the launcher.
    if serverPIDStatus && $1; then
      return 0
    fi
    # The workarea suggests a started server.  Either the script started a
    # server process that ended before removing .sCommand, or the server was
    # started without using the script (so the PID file was never created).
    # In either case, verify the status using the launcher.
    if $1; then
      javaCmd '' "${SERVER_NAME}" $2 > /dev/null
    else
      javaCmd '' "${SERVER_NAME}" --pid="${PID}" $2
    fi
    rc=$?

    if [ $rc = 1 ]; then
      # The launcher verified the server is not running.  Remove the indicator
      # files to avoid checking again on subsequent script invocations.
      rm -f "${SERVER_OUTPUT_DIR}/workarea/.sCommand"
    fi

    return $rc
  fi

  if serverPIDStatus; then
    if $1; then
      return 0
    fi

    # PID file suggests a started server with a removed workspace.  Verify
    # using the launcher.
    javaCmd '' "${SERVER_NAME}" --pid-file="${X_PID_FILE}" --pid="${PID}" $2
    return $?
  fi

  # Neither workarea nor PID file suggest a started server.
  return 1
}

##
## serverPIDStatus: Check the PID file for a running server process.  ${PID}
##                  will be set to the PID if running or to empty if not.
serverPIDStatus()
{
  if [ -f "${X_PID_FILE}" ]; then
    PID=`cat "${X_PID_FILE}"`
    if $PS_P "$PID" > /dev/null 2>&1; then
      return 0
    fi

    rm "${X_PID_FILE}"
  fi

  PID=
  return 1
}

##
## checkServer: Check for created/existing server.  If it does not, issue an
##              error message if the first argument is true, and then exit the
##              script regardless.
checkServer()
{
  if [ -d "${SERVER_CONFIG_DIR}" ] || [ "defaultServer" = "${SERVER_NAME}" ]
  then
    return 0
  fi

  if $1
  then
    javaCmd '' --message:info.serverNotExist "${SERVER_NAME}"
    rc=$?
    if [ $rc = 0 ]
    then
      rc=1
    fi
  else
    rc=1
  fi

  return $rc
}

##
## installEnv: Set variables for a non-server or nonexistent server command.
installEnv()
{
  readServerEnv "${WLP_INSTALL_DIR}/etc/server.env"
  installEnvDefaults
  serverEnvDefaults
}

##
## installEnvDefaults: Set variable defaults for a non-server or nonexistent
##                     server command.
installEnvDefaults()
{
  readServerEnv "${WLP_INSTALL_DIR}/java/java.env"
  readServerEnv "${WLP_INSTALL_DIR}/etc/default.env"

  if [ -z "${WLP_USER_DIR}" ]
  then
    if [ -z "${WLP_DEFAULT_USER_DIR}" ]
    then
      WLP_DEFAULT_USER_DIR=${WLP_INSTALL_DIR}/usr
    fi
    WLP_USER_DIR=${WLP_DEFAULT_USER_DIR}
  fi

  if [ -z "${WLP_OUTPUT_DIR}" ]
  then
    if [ -z "${WLP_DEFAULT_OUTPUT_DIR}" ]
    then
      WLP_DEFAULT_OUTPUT_DIR=${WLP_USER_DIR}/servers
    fi
    WLP_OUTPUT_DIR=${WLP_DEFAULT_OUTPUT_DIR}
  fi

  SERVER_CONFIG_DIR=${WLP_USER_DIR}/servers/${SERVER_NAME}

  if $os400lib
  then
    if [ -z "${WAS_PRODUCT_LIB}" ] || [ -z "${WAS_SHARED_LIB}" ]
    then
      os400lib=false
    fi
  fi

  export WLP_USER_DIR
  export WLP_OUTPUT_DIR
}

##
## serverEnvDefaults: Set defaults for server variables.
serverEnvDefaults()
{
  SERVER_OUTPUT_DIR=${WLP_OUTPUT_DIR}/${SERVER_NAME}

  # Unset set/tested variables to prevent collisions with nested process invocations
  if [ -z "$LOG_DIR" ]
  then
    X_LOG_DIR=${SERVER_OUTPUT_DIR}/logs
  else
    X_LOG_DIR=${LOG_DIR}
    # unset LOG_DIR
  fi
  export X_LOG_DIR

  if [ -z "$LOG_FILE" ]
  then
    X_LOG_FILE=console.log
  else
    X_LOG_FILE=${LOG_FILE}
    unset LOG_FILE
  fi
  export X_LOG_FILE

  if [ -z "$PID_DIR" ]; then
    X_PID_DIR=${WLP_OUTPUT_DIR}/.pid
  else
    X_PID_DIR=${PID_DIR}
    unset PID_DIR
  fi

  if [ -z "$PID_FILE" ]; then
    X_PID_FILE=${X_PID_DIR}/${SERVER_NAME}.pid
  else
    X_PID_FILE=${PID_FILE}
    X_PID_DIR=`dirname "${X_PID_FILE}"`
    unset PID_FILE
  fi

  if [ -z "${JAVA_HOME}" ]
  then
    if [ -z "${JRE_HOME}" ]
    then
      if [ -z "${WLP_DEFAULT_JAVA_HOME}" ]
      then
        # Use whatever java is on the path
        JAVA_CMD=java
      else
        substitutePrefixVar "${WLP_DEFAULT_JAVA_HOME}" WLP_INSTALL_DIR "${WLP_INSTALL_DIR}"
        JAVA_HOME=${substitutePrefixVarResult}
        JAVA_CMD=${JAVA_HOME}/bin/java
      fi
    else
      JAVA_HOME=${JRE_HOME}
      JAVA_CMD=${JRE_HOME}/bin/java
    fi
  else
    if [ -f "${JAVA_HOME}/jre/bin/java" ]
    then
        JAVA_CMD=${JAVA_HOME}/jre/bin/java
    else
        JAVA_CMD=${JAVA_HOME}/bin/java
    fi
  fi

  # Prefer OPENJ9_JAVA_OPTIONS to deprecated IBM_JAVA_OPTIONS
  if [ -z "${OPENJ9_JAVA_OPTIONS}" ]
  then
    SPECIFIED_JAVA_OPTIONS=${IBM_JAVA_OPTIONS}
  else
    SPECIFIED_JAVA_OPTIONS=${OPENJ9_JAVA_OPTIONS}
  fi
  
  # Command-line parsing of -Xshareclasses does not allow "," in cacheDir.
  case ${WLP_OUTPUT_DIR} in
  *,*)
    SERVER_IBM_JAVA_OPTIONS=${SPECIFIED_JAVA_OPTIONS}
    ;;
  *)
    if ( echo "${SPECIFIED_JAVA_OPTIONS}" | grep -q "Xshareclasses"); then
      SERVER_IBM_JAVA_OPTIONS=${SPECIFIED_JAVA_OPTIONS}
    else
      # Set -Xscmx
      if [ "${ACTION}" = "debug" ]
      then
        XSCMX_VAL="130m"
      else
        XSCMX_VAL="80m"
      fi
  
      if $shareclassesCacheDirPerm
      then
        SERVER_IBM_JAVA_OPTIONS="-Xshareclasses:name=liberty-%u,nonfatal,cacheDir=\"${WLP_OUTPUT_DIR}/.classCache\",cacheDirPerm=1000 -XX:ShareClassesEnableBCI -Xscmx${XSCMX_VAL} ${SPECIFIED_JAVA_OPTIONS}"
      else
        SERVER_IBM_JAVA_OPTIONS="-Xshareclasses:name=liberty-%u,nonfatal,cacheDir=\"${WLP_OUTPUT_DIR}/.classCache\" -XX:ShareClassesEnableBCI -Xscmx${XSCMX_VAL} ${SPECIFIED_JAVA_OPTIONS}"
      fi
    fi
  esac

  # Add -Xquickstart -Xshareclasses:none for client JVMs only.  We don't want 
  # shared classes cache created for client operations.
  # Add -Dcom.ibm.tools.attach.enable=yes to allow self-attach on z/OS
  
  IBM_JAVA_OPTIONS="-Xquickstart -Dcom.ibm.tools.attach.enable=yes ${SPECIFIED_JAVA_OPTIONS} -Xshareclasses:none"
  export IBM_JAVA_OPTIONS
  OPENJ9_JAVA_OPTIONS="-Xquickstart -Dcom.ibm.tools.attach.enable=yes ${SPECIFIED_JAVA_OPTIONS} -Xshareclasses:none"
  export OPENJ9_JAVA_OPTIONS

  # Set a default file encoding if needed
  if [ -n "$defaultFileEncoding" ]; then
    if ! expr "${JVM_OPTIONS_QUOTED} ${JVM_ARGS} ${SPECIFIED_JAVA_OPTIONS}" : '.*\(-Dfile\.encoding\=[^[:space:]]\)' > /dev/null; then
      JVM_ARGS="${JVM_ARGS} -Dfile.encoding=$defaultFileEncoding"
    fi
  fi

  serverUmask
}

##
## serverEnv: Set variables for an existing server.
serverEnv()
{
  readServerEnv "${WLP_INSTALL_DIR}/etc/server.env"
  installEnvDefaults

  readServerEnv "${WLP_USER_DIR}/shared/server.env"
  readServerEnv "${SERVER_CONFIG_DIR}/server.env"
  serverEnvDefaults
}

##
## serverEnvAndJVMOptions: Read server.env files and set environment variables.
##                         Read jvm.options file into ${JVM_OPTIONS_QUOTED}
serverEnvAndJVMOptions()
{
  serverEnv

  # Avoids HeadlessException on all platforms and Liberty JVMs appearing as applications and stealing focus on Mac.
  # Declare allowAttachSelf=true so that the VM will permit a late self attach if localConnector-1.0 is enabled
  # need -Dcom.ibm.tools.attach.enable=yes for java 11 on z/OS
  if [ "${uname}" = OS/390 ]; then
    JVM_OPTIONS_QUOTED="-Djava.awt.headless=true -Djdk.attach.allowAttachSelf=true -Dcom.ibm.tools.attach.enable=yes"
  else
    JVM_OPTIONS_QUOTED="-Djava.awt.headless=true -Djdk.attach.allowAttachSelf=true"
  fi
  SERVER_JVM_OPTIONS_QUOTED=${JVM_OPTIONS_QUOTED}

  rc=0


  # The order of merging the jvm.option files sets the precedence.
  # Once a given jvm option is set, it will be overridden if a duplicate
  # is seen later. They will both be written in to the options parameter
  # but the last one written will take precedence.  If none are read
  # the script will try to read the jvm.options in etc

  TEMP_SAVED_OPTIONS=$SERVER_JVM_OPTIONS_QUOTED
  mergeJVMOptions "${WLP_USER_DIR}/shared/jvm.options"
  if [ $rc -ne 0 ]; then
    return $rc
  fi  
  # This location is intentionally not documented  but removing might break existing installations
  mergeJVMOptions "${WLP_INSTALL_DIR}/usr/shared/jvm.options"
  if [ $rc -ne 0 ]; then
    return $rc
  fi
  mergeJVMOptions "${SERVER_CONFIG_DIR}/configDropins/defaults/jvm.options"
  if [ $rc -ne 0 ]; then
    return $rc
  fi
  mergeJVMOptions "${SERVER_CONFIG_DIR}/jvm.options"
  if [ $rc -ne 0 ]; then
    return $rc
  fi
  mergeJVMOptions "${SERVER_CONFIG_DIR}/configDropins/overrides/jvm.options"
  if [ $rc -ne 0 ]; then
    return $rc
  fi

  # If none of the four files above are seen we will check for an options
  # file in the etc folder.
  if [ $READ_ETC -eq 1 ]; then
    SERVER_JVM_OPTIONS_QUOTED=$TEMP_SAVED_OPTIONS
    mergeJVMOptions "${WLP_INSTALL_DIR}/etc/jvm.options"
  fi

  # Determine location of Java Platform Module System (JPMS) modules directory
  if [ -z "${JAVA_HOME}" ]; then

    # JAVA_HOME not set.  Locate java using the PATH.
    JAVA_PATH=$(command -v java)
    if [ -n "${JAVA_PATH}" ] ; then

      REAL_JAVA_PATH=$(findRealPath "${JAVA_PATH}")

      if [ -n "${REAL_JAVA_PATH}" ]; then
         JPMS_MODULE_FILE_LOCATION=$(dirname "$(dirname "${REAL_JAVA_PATH}")")/lib/modules
      fi
    fi
  else
    JPMS_MODULE_FILE_LOCATION="${JAVA_HOME}/lib/modules"
  fi

  # If we are running on Java 9 and up, apply Liberty's built-in java 9 options  
  if [ -f "${JPMS_MODULE_FILE_LOCATION}" ]; then
    mergeJVMOptions "${WLP_INSTALL_DIR}/lib/platform/java/java9.options"
  fi

  return $rc
}

# Resolve symbolic links in a path iteratively
findRealPath() {
  path="$1"

  # Try realpath command
  if command -v realpath >/dev/null; then
    resolvedPath=$(realpath "${path}" 2>/dev/null)
  fi
  
  # If realpath is not available, try readlink -e
  if [ -z "${resolvedPath}" ] && command -v readlink >/dev/null; then
    resolvedPath=$(readlink -e "${path}" 2>/dev/null)
  fi

  # If neither of those methods work, try a more basic approach
  if [ -z "${resolvedPath}" ]; then 
    symLinkCount=0

    # Loop ends when resolvedPath does not contain any symlinks  
    while true; do

      resolvedPath=""
      IFS="/" # Set the delimiter to '/'
      # Split the path into components.  Purposefully not putting quotes around $path.
      set -- $path
      pathContainsSymLink="false"
      for component; do
	  
        if [ -z "$component" ]; then
          continue # Skip empty components
        fi

        # Combine the resolved path & component
        if [ -z "$resolvedPath" ]; then
          case "$path" in
          /*)
            resolvedPath="/$component"
            ;;
          *) # If path is relative, then don't add leading slash
            resolvedPath=$component
            ;;
          esac
        else
          resolvedPath="$resolvedPath/$component"
        fi

        if [ -L "$resolvedPath" ]; then
          pathContainsSymLink="true"
          symLinkCount=$((symLinkCount + 1))
		  
          # Get the target of the symbolic link
          target=$(ls -l "$resolvedPath" | awk '{print $NF}')

          # Don't lose the remaining path past the symlink
          if [ "$resolvedPath" = "$path" ]; then
            remaining_path=""
          else
            # Remove the resolved part from the original path
            remaining_path="${path#"${resolvedPath%/}"/}"
          fi

          # Save the prefix up one level from the symlink
          savePrefix=$(cd "$(dirname "$resolvedPath")" && pwd)
		  
          case "$target" in
          /*)
            # Absolute path case
            resolvedPath="$target/$remaining_path"
            ;;
          *)
            # Relative path case"
            resolvedPath="$savePrefix/$target/$remaining_path"
            ;;
          esac

          break
        else
          :  # $resolvedPath is  NOT a symLink
        fi
      done  # end for

      if [ $pathContainsSymLink = "false" ]; then
        break
      fi

      if [ $symLinkCount -gt 100  ]; then
        issueMessage --message:warning.tooManySymbolicLinks "$1"
        resolvedPath=""
        break
      fi

      path=$resolvedPath
    done  # end while
  fi

  # Return the fully resolved path
  echo "$resolvedPath"
}

mergeJVMOptions()
{
    jvmOptions=$1

    if [ -f "$jvmOptions" ]; then
      READ_ETC=0
      saveIFS=$IFS
      IFS=$newline

      for option in `readNativeFile "$jvmOptions" '[#-]' | LC_ALL=C tr -d '\r'`; do
        if [ -n "$option" ]; then
          case $option in
          \#*);;
          *)
            escapeForEval "${option}"
            SERVER_JVM_OPTIONS_QUOTED="${SERVER_JVM_OPTIONS_QUOTED} '${option}'"
            ;;
          esac
        fi
      done
      IFS=$saveIFS
    fi

}

##
## readServerEnv: Read server.env file and export environment variables.
readServerEnv()
{
  # By default, we do not attempt to resolve variables in server.env
  variableExpansionEnabled=0
  if [ -f "$1" ]; then
    saveIFS=$IFS
    IFS=$newline
    for line in `readNativeFile "$1" '[#_A-Za-z=]' | tr -d '\r'`; do
      case $line in

      \#*)
           # line is a comment.
           # A comment line enables variable expansion if it contains the keyword,
           # "enable_variable_expansion", and nothing else (ignoring case and white space).
           # If found, we use the source command ( . ) which performs all of the server.env
           # variable assignments while resolving any variable references.
           if [ $variableExpansionEnabled = 0  ] && [ $(safeEcho $line | grep -i "enable_variable_expansion") ]
           then
              processedLine=$(safeEcho $line | tr -d '[:blank:]' | tr '[:upper:]' '[:lower:]')
              if [ "#enable_variable_expansion" = $processedLine ]
              then
                 eval ". $1"
                 variableExpansionEnabled=1
              fi
           fi ;;
      *=*)
          # The sed commands extracts the variable from the line but only if it contains alpha-numeric
          # characters. Only accept alphanumeric variable names to avoid eval complexities.
          if [ "$uname" != "SunOS" ] && [ "$uname" != "OS/390" ]; then

            # This is the preferred pattern, because it handles characters 
            # from multiple languages, but it doesn't work on Solaris.
            var=$(safeEcho "$line" | sed -n -e 's/^\([_[:alpha:]][_[:alnum:]]*\)=.*/\1/p')

          else
            # On Solaris & Z/OS we cannot use alpha or alnum
            var=$(safeEcho "$line" | awk -F= '{print $1}' | sed '/^[a-zA-Z_][a-zA-Z0-9_]*$/!d')
          fi

          if [ -n "$var" ]; then

            case $var in
            *=*)
              SERVER_ENV_SETUP_FAILURE="${var} ${SERVER_ENV_SETUP_FAILURE}" ;;
            *)
              # if variable expansion is enabled, then we have already "sourced" the entire file.  As a result
              # all of the variables in server.env have already been defined as shell variables.  To make them
              # accessible to child processes, they need to be exported.
              if [ $variableExpansionEnabled = 1 ]
              then
                 eval "export ${var}"
              else
                 # Variable expansion is not enabled.  Just set the variable to whatever is on the right side 
                 # of the assignment and export it.  If a later line enables variable expansion, the source
                 # command will reassign the variable (which is already exported) to the resolved value.
                 extractValueAndEscapeForEval "${line}"
                 eval "${var}=${escapeForEvalResult}; export ${var}"
              fi
            esac
          fi ;;
      esac
    done
    IFS=$saveIFS
  fi
}

## Set the environment varialbes needed by default for criu to work.
## Mostly these are variables read by the OS libraries to ensure a portable process
criuEnvDefaults() {
  if [ -z "$GLIBC_TUNABLES" ]
  then
    GLIBC_TUNABLES=glibc.cpu.hwcaps=z14,arch_2_07,-XSAVEC,-XSAVE,-AVX2,-ERMS,-AVX,-AVX_Fast_Unaligned_Load:glibc.pthread.rseq=0
  else
    GLIBC_TUNABLES=${GLIBC_TUNABLES}:glibc.cpu.hwcaps=z14,arch_2_07,-XSAVEC,-XSAVE,-AVX2,-ERMS,-AVX,-AVX_Fast_Unaligned_Load:glibc.pthread.rseq=0
  fi
  export GLIBC_TUNABLES
  export LD_BIND_NOT=on

  # Disable CRIU's use of HW breakpoints on restore.
  # HW breakpoints are slower in some environments.
  export CRIU_FAULT=130 # 130=FI_NO_BREAKPOINTS

  # for z/OS arch s390x we need to set DFLTCC=0 to disable
  # hardware-accelerated compression for checkpoint
  CHECK_ARCH=`arch`
  if [ "$CHECK_ARCH" = "s390x" ]; then
    export DFLTCC=0
  fi
}

##
## Detects the code page of the file and converts to EBCDIC,
## if necessary, before cat'ing.
##
## Only applicable if running in a native EBCDIC environment (z/OS).
##
## $1 the file name
## $2 pattern denoting the expected first char of file
readNativeFile() {
  if ${nativeEBCDIC}; then
    # ASCII 'm' overlaps with EBCDIC '_', so strip it out before detecting the codepage.
    # Note: cat used here to handle ASCII-tagged files.
    filecontents=`cat "$1" | iconv -f ISO8859-1 -t IBM-1047 | tr -d 'm\r\n'`
    case $filecontents in
      $2*) iconv -f ISO8859-1 -t IBM-1047 "$1" ;;  # ASCII file.
      *) cat "$1"                # EBCDIC file or ASCII-tagged file.
    esac
  else
    cat "$1"
  fi
}

##
## serverWorkingDirectory: Change the JVM working directory to ${SERVER_WORKING_DIR}
serverWorkingDirectory()
{
  if [ -z "${SERVER_WORKING_DIR}" ]; then
    SERVER_WORKING_DIR=$SERVER_OUTPUT_DIR
  else
    case "${SERVER_WORKING_DIR}" in
      /*) len=${#SERVER_WORKING_DIR}
          if [ $len -eq 1 ]; then
            SERVER_WORKING_DIR=$SERVER_OUTPUT_DIR 
          fi ;;
      *) SERVER_WORKING_DIR="${SERVER_OUTPUT_DIR}/${SERVER_WORKING_DIR}" ;;
    esac
  fi

  mkdirs "${SERVER_WORKING_DIR}"
  cd "${SERVER_WORKING_DIR}"
}

## set the umask value to remove all permissions for the Other category
## but leave existing values for User and Group unaltered
##
## If the WLP_SKIP_UMASK variable is set then do not set umask
serverUmask()
{
  if [ -z "${WLP_SKIP_UMASK}" ]; then
    $UMASK_O
  fi
}

##  Silently kill the process associated with pid passed in as argument.
##  No output if the pid does not exist.
##
killSilent() {
  kill ${1} > /dev/null 2>&1
}
##
## javaCmd: Execute a java-based command.  Arguments are:
##
##   backgroundLogFile
##     Non-empty path for output log if the process should be created in the
##     background; empty otherwise.  Directory must exist.
##
##   "$@"
##     All remaining arguments to be passed to the command
##
## The following variables must be set:
##   ${JAVA_CMD} - the java command
##   ${JAVA_DEBUG} - JVM options to enable debug, or empty
##   ${JAVA_AGENT_QUOTED} - The -javaagent argument quoted for eval
##   ${JVM_OPTIONS_QUOTED} - jvm.options quoted for eval
##   ${WLP_INSTALL_DIR_QUOTED} - ${WLP_INSTALL_DIR} quoted for eval
javaCmd()
{
  JAVA_CMD_LOG=$1
  shift
  
  # Filter all of the -D and -X arguments off of the argument line and add them to $JVM_OPTIONS_QUOTED
  REMAINING_ARGS=""
  for option in "$@"; do
    if [ -n "${option}" ]; then
      if [ -z "${option##-D*}" ] || [ -z "${option##-X*}" ]; then
        JVM_OPTIONS_QUOTED="$JVM_OPTIONS_QUOTED '$option'"
      elif [ $ACTION = "checkpoint" ] && [ -z "${option##-agentlib:*}" ]; then
        # we allow debug agentlib for checkpoint only
        JVM_OPTIONS_QUOTED="$JVM_OPTIONS_QUOTED '$option'"
      elif [ $ACTION = "checkpoint" ] && [ -z "${option##--at=*}" ]; then
        CHECKPOINT_AT=${option#--at=}
      else
        if [ -z "$REMAINING_ARGS" ] ; then
          REMAINING_ARGS="$option"
        else
          REMAINING_ARGS="$REMAINING_ARGS $option"
        fi  
      fi
    else
      REMAINING_ARGS="$REMAINING_ARGS $option"
    fi
  done
  
  if [ $ACTION = "checkpoint" ]; then
    if [ -z "$CHECKPOINT_AT" ]; then
      CHECKPOINT_AT="beforeAppStart"
    fi
    REMAINING_ARGS="$REMAINING_ARGS --internal-checkpoint-at=$CHECKPOINT_AT"

    # Default list of immutable env vars for checkpoint
    X_WLP_IMMUTABLE_VARS="-Dorg.eclipse.openj9.criu.ImmutableEnvVars=INVOKED,WLP_USER_DIR,WLP_OUTPUT_DIR,LOG_DIR,X_LOG_DIR,LOG_FILE,X_LOG_FILE,VARIABLE_SOURCE_DIRS,X_CMD"

    # Add any additional user specified variables
    if [ -n "$WLP_IMMUTABLE_VARS" ]; then
      X_WLP_IMMUTABLE_VARS="$X_WLP_IMMUTABLE_VARS,$WLP_IMMUTABLE_VARS"
    fi

    # IgnoreUnrecognizedVMOptions is used so that other JVMs don't fail on these args
    # IgnoreUnrecognizedRestoreOptions to avoid failing to restore if unsupported options are specified to restore
    JVM_OPTIONS_QUOTED="$JVM_OPTIONS_QUOTED -XX:+IgnoreUnrecognizedVMOptions -XX:+IgnoreUnrecognizedRestoreOptions -XX:+EnableCRIUSupport -XX:+CRIURestoreNonPortableMode $X_WLP_IMMUTABLE_VARS"


    checkCriuUnprivileged
    if [ $DO_CRIU_UNPRIVILEGED = true ]; then
      JVM_OPTIONS_QUOTED="$JVM_OPTIONS_QUOTED -Dio.openliberty.checkpoint.criu.unprivileged=true"
    fi
    if [ -n "$CRIU_LOG_LEVEL" ]; then
      # A system property is used here to avoid having to read env on the 
      # checkpoint side. If reading the env from the Java checkpoint side we need
      # to make the env an immutable value but we don't want to force the
      # same CRIU_LOG_LEVEL on both the checkpoint and restore side here.
      JVM_OPTIONS_QUOTED="$JVM_OPTIONS_QUOTED -Dio.openliberty.checkpoint.criu.loglevel=$CRIU_LOG_LEVEL"
    fi
    criuEnvDefaults
  fi
  

  # Set all the parameters for the java command.  We use eval so that each line
  # in jvm.options is treated as a distinct argument.
  eval "set -- ${JAVA_AGENT_QUOTED} ${JVM_OPTIONS_QUOTED} ${JAVA_DEBUG} ${JVM_ARGS} -jar ${WLP_INSTALL_DIR_QUOTED}/bin/tools/ws-server.jar ""$REMAINING_ARGS"
  ARGS="$@"

  if [ $ACTION = "checkpoint" ]; then
    if [ -d "${SERVER_OUTPUT_DIR}/workarea/checkpoint" ]; then
      rm -rf ${SERVER_OUTPUT_DIR}/workarea/checkpoint
      rc=$? 
      if [ $rc -ne 0 ]; then
         issueMessage --message:error.checkpoint.delete "${SERVER_OUTPUT_DIR}/workarea/checkpoint"
         return $rc
      fi   
    fi
  fi

  # Do not add extra logic after the commands without preserving $?
  if [ -n "${JAVA_CMD_LOG}" ]; then
    rmIfExist "${JAVA_CMD_LOG}"
    "${JAVA_CMD}" "$@" >> "${JAVA_CMD_LOG}" 2>&1 &
  elif [ $ACTION = "run" ] || [ $ACTION = "debug" ]; then
    exec "${JAVA_CMD}" "$@"
  elif [ $ACTION = "checkpoint" ] && [ -f "${SERVER_CONFIG_DIR}/server.xml" ]; then
    # Tail is used to print the content of the log file to the console
    # output. This is done so we can restore in the foreground (run)
    # and in the background (start)
    CHECKPOINT_CONSOLE_LOG="${X_LOG_DIR}/${X_LOG_FILE}"
    # remove existing console log and the possible backup for checkpoint
    rmIfExist "${CHECKPOINT_CONSOLE_LOG}"
    rmIfExist "${X_LOG_DIR}/checkpoint-${X_LOG_FILE}"
    mkdirs "${X_LOG_DIR}"
    # touch the log so we can tail it immediately
    touchIfNotExist "${CHECKPOINT_CONSOLE_LOG}"
    tail -f "${CHECKPOINT_CONSOLE_LOG}" &
    TAIL_PID=$!
    # trap on the signals here to ensure the background tail process ends.
    trap "exit" INT TERM
    trap "killSilent $TAIL_PID" EXIT
    #On some shells (bash) the shell will issue an ugly message to STDERR when criu kills the process.
    # We direct STDERR to /dev/null to avoid that noise.
    {
      # JAVA_CHECKPOINT_PREFIX_CMD is used externally to allow prefixing commands before the
      # java invocation.  For example, to make calls to ld.so to fine tune glibc behavior.
      #  Launch with stdin set to /dev/null to avoid file-handle complications on criu restore
      ${JAVA_CHECKPOINT_PREFIX_CMD}"${JAVA_CMD}" "$@" >> "${CHECKPOINT_CONSOLE_LOG}" 2>&1 < /dev/null
    } 2> /dev/null
    rc=$?
    if [ $rc -eq 20 ]; then
      # bad checkpoint arg; sleep to allow the message to go to tail
      sleep 2
    fi
    if [ $rc -eq 74 ] && [ -f "${X_LOG_DIR}/checkpoint/checkpoint.log" ]; then
      issueMessage --message:error.checkpoint.failed "${X_LOG_DIR}/checkpoint/checkpoint.log"
      cat "${X_LOG_DIR}/checkpoint/checkpoint.log"
      # check if selinux is preventing net_diag feature in criu
      CRIU_UNPRIVILEGED_ARGS=""
      if [ $DO_CRIU_UNPRIVILEGED = true ]; then
        CRIU_UNPRIVILEGED_ARGS="--unprivileged"
      fi
      criu "$CRIU_UNPRIVILEGED_ARGS" check "--feature=net_diag_raw" > /dev/null 2>&1
      ckrc=$?
      if [ $ckrc -ne 0 ]; then
         issueMessage --message:error.checkpoint.failed.selinux.netdiag "${SERVER_OUTPUT_DIR}/workarea"
      fi
    fi
    kill $TAIL_PID
    # Above JAVA_CMD should terminate after a criu dump and SIGKILL. 
    # Check for expected return code of 137 from the KILL.
    if [ $rc -eq 137 ]; then
      backupWorkarea
      #Reset rc. If backupWorkarea returns 0 then overall checkpoint action is successful.
      rc=$?
      if [ $rc -ne 0 ]; then
         issueMessage --message:error.checkpoint.workarea.backup "${SERVER_OUTPUT_DIR}/workarea"
      fi
      rm -f "${SERVER_OUTPUT_DIR}/workarea/.sCommand"
    fi
    return $rc
  else
    "${JAVA_CMD}" "$@"
  fi
}

##
## Issue a translated warning message. The java command will use the same options as javaCmd,
## but will ignore debug settings and will not exit after running the command when 'run'
## or 'debug' is used
##
##     "$@"
##       All arguments passed to the java command
##
issueMessage()
{
  # Set all the parameters for the java command.  We use eval so that each line
  # in jvm.options is treated as a distinct argument.
  eval "set -- ${JAVA_AGENT_QUOTED} ${JVM_OPTIONS_QUOTED} ${JVM_ARGS} -jar ${WLP_INSTALL_DIR_QUOTED}/bin/tools/ws-server.jar "'"$@"'
  ARGS="$@"

  "${JAVA_CMD}" "$@"
}



##
## serverCmdOS400: Launch a server process using the OS400 native launcher.
##
##   backgroundLogFile
##     Non-empty path for output log if the process should be created in the
##     background; empty otherwise.  Directory must exist.
##
##   "$@"
##     All remaining arguments to be passed to the command
##
## The following variables must be set:
##   ${JAVA_CMD} - the java command
##   ${JAVA_DEBUG} - JVM options to enable debug, or empty
##   ${JAVA_AGENT_QUOTED} - The -javaagent argument quoted for eval
##   ${JVM_OPTIONS_QUOTED} - jvm.options quoted for eval
##   ${WLP_INSTALL_DIR_QUOTED} - ${WLP_INSTALL_DIR} quoted for eval
serverCmdOS400()
{
  SERVER_CMD_OS400_BACKGROUND_LOG=$1
  shift

  eval "set -- -jvmargs ${JAVA_AGENT_QUOTED} ${JVM_OPTIONS_QUOTED} ${JAVA_DEBUG} ${JVM_ARGS} -jar ${WLP_INSTALL_DIR_QUOTED}/bin/tools/ws-server.jar "'"$@"'

  export JAVA_HOME

  if [ -n "${SERVER_CMD_OS400_BACKGROUND_LOG}" ]
  then
    "/QSYS.LIB/${WAS_PRODUCT_LIB}.LIB/QWLPSTRSVR.PGM" \
      -pgm "${WAS_PRODUCT_LIB}/QWLPSTRSVR" \
      -server "${SERVER_NAME}" \
      -sharedlib "${WAS_SHARED_LIB}" \
      -pidfile "${X_PID_FILE}" \
      -background \
      -logdir  "${X_LOG_DIR}"  \
      -serveroutputdir "${SERVER_OUTPUT_DIR}" \
      -serverlog "${SERVER_CMD_OS400_BACKGROUND_LOG}" \
      "$@"
  else
    "/QSYS.LIB/${WAS_PRODUCT_LIB}.LIB/QWLPSTRSVR.PGM" \
      -logdir  "${X_LOG_DIR}"  \
      -serveroutputdir "${SERVER_OUTPUT_DIR}" \
      "$@"
  fi
}

##
## serverCmd: Launch a server process.
##
##   backgroundLogFile
##     Non-empty path for output log if the process should be created in the
##     background; empty otherwise.  Directory must exist.
##
## PID will be set if the process is launched in the background and the return
## code is 0.
##
serverCmd()
{

   ## If one or more variables in server.env contains non-alphanumeric characters
   ## SERVER_ENV_SETUP_FAILURE will contain the invalid entries. We have to
   ## temporarily set ACTION to 'start' so that the process will not exit after
   ## the error message when ACTION is 'run' or 'debug'
   if [ -n "${SERVER_ENV_SETUP_FAILURE}" ];
   then
	for i in ${SERVER_ENV_SETUP_FAILURE}; do
            issueMessage --message:warning.variable.invalid "$i"
	done
   fi

  SERVER_CMD_BACKGROUND_LOG=$1
  shift

  ## On OS400, we need to create the defaultServer first from this script
  ## to get all the permissions correct. If we allow the server start
  ## to inline the server create from the java process, we won't get
  ## the correct permissions
  if $os400lib && [ ! -d "${SERVER_CONFIG_DIR}" ] && [ "defaultServer" = "${SERVER_NAME}" ]
  then
    # Create the server or exit the script
    createServer || exit $?
  fi

  SAVE_JVM_OPTIONS_QUOTED=${JVM_OPTIONS_QUOTED}
  JVM_OPTIONS_QUOTED=${SERVER_JVM_OPTIONS_QUOTED}
  
  SAVE_IBM_JAVA_OPTIONS=${IBM_JAVA_OPTIONS}
  IBM_JAVA_OPTIONS=${SERVER_IBM_JAVA_OPTIONS}
  SAVE_OPENJ9_JAVA_OPTIONS=${OPENJ9_JAVA_OPTIONS}
  OPENJ9_JAVA_OPTIONS=${SERVER_IBM_JAVA_OPTIONS}

  if $os400lib; then
    serverCmdOS400 "${SERVER_CMD_BACKGROUND_LOG}" "${SERVER_NAME}" "$@"
    rc=$?

    JVM_OPTIONS_QUOTED=${SAVE_JVM_OPTIONS_QUOTED}
    IBM_JAVA_OPTIONS=${SAVE_IBM_JAVA_OPTIONS}
    OPENJ9_JAVA_OPTIONS=${SAVE_OPENJ9_JAVA_OPTIONS}

    if [ -n "${SERVER_CMD_BACKGROUND_LOG}" ]; then
      PID=`cat "${X_PID_FILE}"`

      # Verify/wait for the process to start
      javaCmd '' "${SERVER_NAME}" --pid="${PID}" --status:start "$@"
      rc=$?
    fi

  elif [ "${uname}" = OS/390 ] && [ -n "${WLP_ZOS_PROCEDURE}" ] && [ -e "${WLP_INSTALL_DIR}/lib/native/zos/s390x/bbgzcsl" ]; then
      # Start the STC procedure
      if [ -n "${WLP_ZOS_JOBNAME}" ]; then
        "${WLP_INSTALL_DIR}/lib/native/zos/s390x/bbgzcsl" "${WLP_ZOS_PROCEDURE}" "${SERVER_NAME}" "${X_PID_FILE}" "${WLP_ZOS_JOBNAME}" "$@"
      else
          "${WLP_INSTALL_DIR}/lib/native/zos/s390x/bbgzcsl" "${WLP_ZOS_PROCEDURE}" "${SERVER_NAME}" "${X_PID_FILE}" "$@"
      fi

      rc=$?

      JVM_OPTIONS_QUOTED=${SAVE_JVM_OPTIONS_QUOTED}
      IBM_JAVA_OPTIONS=${SAVE_IBM_JAVA_OPTIONS}
      OPENJ9_JAVA_OPTIONS=${SAVE_OPENJ9_JAVA_OPTIONS}

      if [ $rc = 0 ]; then
        # Verify/wait for the process to start
        PID=`cat "${X_PID_FILE}"`
        javaCmd '' "${SERVER_NAME}" --pid="${PID}" --status:start "$@"
        rc=$?

      elif [ $rc = 20 ]; then
        # PID file not created
        javaCmd '' --message:error.zosProcStart.create.pidfile "${SERVER_NAME}" "${WLP_ZOS_PROCEDURE}"
        rc=$?
        if [ $rc = 0 ]; then
           rc=22
        fi

      elif [ $rc = 21 ]; then
        # Command has exceeded the max allowable length for MGCRE
        javaCmd '' --message:error.zosProcStart.start.length "${WLP_ZOS_PROCEDURE}"
        rc=$?
        if [ $rc = 0 ]; then
           rc=22
        fi

      elif [ $rc = 22 ]; then
        # The specified z/OS procedure name is invalid
        javaCmd '' --message:error.zosProcStart.procedure.invalid "${WLP_ZOS_PROCEDURE}"
        rc=$?
        if [ $rc = 0 ]; then
           rc=22
        fi

      elif [ $rc = 23 ]; then
        # The specified z/OS jobname is invalid
        javaCmd '' --message:error.zosProcStart.jobname.invalid "${WLP_ZOS_PROCEDURE}" "${WLP_ZOS_JOBNAME}"
        rc=$?
        if [ $rc = 0 ]; then
           rc=22
        fi

      else
        # Problem issuing MGCRE
        javaCmd '' --message:error.zosProcStart.mvs.start "${WLP_ZOS_PROCEDURE}" "$rc"
        rc=$?
        if [ $rc = 0 ]; then
           rc=22
        fi
      fi

  
  else
    X_CMD="${JAVA_CMD} ${ARGS}"
    export X_CMD

    javaCmd "${SERVER_CMD_BACKGROUND_LOG}" "${SERVER_NAME}" "$@"
    rc=$?
      
    PID=$!

    JVM_OPTIONS_QUOTED=${SAVE_JVM_OPTIONS_QUOTED}
    IBM_JAVA_OPTIONS=${SAVE_IBM_JAVA_OPTIONS}
    OPENJ9_JAVA_OPTIONS=${SAVE_OPENJ9_JAVA_OPTIONS}

    if [ -n "${SERVER_CMD_BACKGROUND_LOG}" ]; then
      # Verify/wait for the process to start
      javaCmd '' "${SERVER_NAME}" --pid="${PID}" --status:start "$@"
      rc=$?
      # write the pid file if return is OK or UNKNOWN
      if [ $rc = 0 -o $rc = $SERVER_UNKNOWN_STATUS ]; then
          safeEcho "${PID}" > "${X_PID_FILE}"
      fi        
    fi
  fi
  return $rc
}

# Create workarea checkpoint (a workarea backup) after criu has checkpointed the JVM and halted the server process.
#  The final layout is
#    ${SERVER_OUTPUT_DIR}/workarea/checkpoint/image/
#    ${SERVER_OUTPUT_DIR}/workarea/checkpoint/workarea/
#  where checkpoint/image is the folder used by criu to write/read process dumps.
#        checkpoint/workarea contains the backup of workarea contents, less the checkpoint dir
backupWorkarea()
{
  # first remove the temporary destination to ensure we get errors if it exists and cannot be deleted
  rm -fr "${SERVER_OUTPUT_DIR}/checkpoint"
  rc=$?
  if [ $rc -ne 0 ]; then
    return 1
  fi

  #  ${SERVER_OUTPUT_DIR}/workarea/checkpoint contains the new criu image dump. Temporarily move it while copying rest of workarea.
  mv -f "${SERVER_OUTPUT_DIR}/workarea/checkpoint" "${SERVER_OUTPUT_DIR}"
  rc=$?
  if [ $rc -ne 0 ]; then
    return 2
  fi

  # copy the workarea to the checkpoint folder so it can be restored later
  cp -r --preserve=all "${SERVER_OUTPUT_DIR}/workarea" "${SERVER_OUTPUT_DIR}/checkpoint/workarea"
  rc=$?
  if [ $rc -ne 0 ]; then
    return 3
  fi

  # mv checkpoint directory back to the original location (now with a copy of the workarea inside)
  mv -f "${SERVER_OUTPUT_DIR}/checkpoint" "${SERVER_OUTPUT_DIR}/workarea/"
  rc=$?
  if [ $rc -ne 0 ]; then
    return 4
  fi

  return $rc
}

restoreWorkarea()
{
  if [ ! -f "${SERVER_OUTPUT_DIR}/workarea/checkpoint/.restoreMarker" ]; then
    return 0
  fi
  if [ ! -d "${SERVER_OUTPUT_DIR}/workarea/checkpoint/workarea" ]; then
    return 1
  fi
  # remove everything but the checkpoint directory; temporarily move checkpoint while removing
  mv "${SERVER_OUTPUT_DIR}/workarea/checkpoint" "${SERVER_OUTPUT_DIR}"
  rc=$?
  if [ $rc -ne 0 ]; then
    return 3
  fi
  rm -rf "${SERVER_OUTPUT_DIR}/workarea"
  # copy backup to workarea
  cp -r --preserve=all "${SERVER_OUTPUT_DIR}/checkpoint/workarea" "${SERVER_OUTPUT_DIR}/workarea"
  rc=$?
  # mv checkpoint directory back
  mkdirs "${SERVER_OUTPUT_DIR}/workarea/"
  mv "${SERVER_OUTPUT_DIR}/checkpoint" "${SERVER_OUTPUT_DIR}/workarea/"
  if [ $rc -ne 0 ]; then
    return 4
  fi
  rc=$?
  rmIfExist "${SERVER_OUTPUT_DIR}/workarea/checkpoint/.restoreMarker"
    if [ $rc -ne 0 ]; then
    return 5
  fi
}

##           restoreInBackground
##               If true the restore command will start in the background
criuRestore()
{
  restoreWorkarea
  rc=$?
  if [ $rc -ne 0 ]; then
    issueMessage --message:error.checkpoint.workarea.restore "${SERVER_OUTPUT_DIR}/workarea/checkpoint/workarea"
    return $rc
  fi 

  BACKGROUND_RESTORE=$1
  if [ -z "${CRIU_LOG_LEVEL}" ]; then
    CRIU_LOG_LEVEL=2
  fi  

  checkCriuUnprivileged
  if [ $DO_CRIU_UNPRIVILEGED = true ]; then
    CRIU_EXTRA_ARGS="--unprivileged $CRIU_EXTRA_ARGS"
  fi

  if [ -z "${CRIU_RESTORE_PATH}" ]; then
    # default to using the criu available on the PATH
    CRIU_RESTORE_PATH=criu
  fi

  if $BACKGROUND_RESTORE
  then
    mkdirs "${X_PID_DIR}"
    rmIfExist "${X_PID_FILE}"
    $CRIU_RESTORE_PATH restore --skip-file-rwx-check --cpu-cap=none --file-locks --tcp-established --images-dir=${SERVER_OUTPUT_DIR}/workarea/checkpoint/image \
      --shell-job --verbosity=${CRIU_LOG_LEVEL} --log-file ${X_LOG_DIR}/checkpoint/restore.log ${CRIU_EXTRA_ARGS} \
      --pidfile ${X_PID_FILE} \
      --restore-detached
    rc=$?
    PID=`cat ${X_PID_FILE}`
    if [ $rc = 0 ]; then
      # successful restore, now check server status
      JVM_OPTIONS_QUOTED=${SAVE_JVM_OPTIONS_QUOTED}
      IBM_JAVA_OPTIONS=${SAVE_IBM_JAVA_OPTIONS}
      OPENJ9_JAVA_OPTIONS=${SAVE_OPENJ9_JAVA_OPTIONS}

      # Verify/wait for the process to start
      javaCmd '' "${SERVER_NAME}" --pid="${PID}" --status:start
      rc=$?
      # write the pid file if return is OK or UNKNOWN
      if [ $rc = 0 -o $rc = $SERVER_UNKNOWN_STATUS ]; then
        safeEcho "${PID}" > "${X_PID_FILE}"
      fi
    fi        
  else
    # For checkpoint we always write to the log file.
    # Use tail to bring the content to the console output
    tail -s0.1 -f "${RESTORE_CONSOLE_LOG}" &
    TAIL_PID=$!
    # trap on the signals here to ensure the background tail process ends
    trap "exit" INT TERM
    trap "killSilent $TAIL_PID" EXIT

    $CRIU_RESTORE_PATH restore --skip-file-rwx-check --cpu-cap=none --file-locks --tcp-established --images-dir=${SERVER_OUTPUT_DIR}/workarea/checkpoint/image \
      --shell-job --verbosity=${CRIU_LOG_LEVEL} --log-file ${X_LOG_DIR}/checkpoint/restore.log ${CRIU_EXTRA_ARGS}
    rc=$?

    kill $TAIL_PID
  fi

  return $rc
}

## restoreServer: restore a server from a checkpoint image if it exists
##

restoreServer()
{
  BACKGROUND_RESTORE=$1

  if [ "${ACTION}" = "checkpoint" ] || [ ! -d "${SERVER_OUTPUT_DIR}/workarea/checkpoint/image" ] ; then
    return 104
  fi

  # must check for --clean option before restoring
  for option in "$@"; do
    if [ -n "${option}" ]; then
      if [ -z "${option##--clean}" ]; then
        # never restore if --clean is specified
        return 104
      fi
    fi
  done

  # restore artifacts and environment prior to checkpoint here
    
  # no good way to check for criu support since fast startup is imperative
  #   just let criu invocation fail with a hopefully informative error message.
  mkdirs ${X_LOG_DIR}/checkpoint

  # save off the log during checkpoint and start fresh from restore 
  RESTORE_CONSOLE_LOG="${X_LOG_DIR}/${X_LOG_FILE}"
  CHECKPOINT_CONSOLE_LOG="${X_LOG_DIR}/checkpoint-${X_LOG_FILE}"
  backupIfNeededOrRemove $RESTORE_CONSOLE_LOG $CHECKPOINT_CONSOLE_LOG
  mkdirs "${X_LOG_DIR}"
  touchIfNotExist "${RESTORE_CONSOLE_LOG}"
 
  # record the start of restoreTime
  echo `date +%s%3N` > ${SERVER_OUTPUT_DIR}/workarea/checkpoint/restoreTime

  criuEnvDefaults

  # Store the current restored environment into a file that the JVM can read to restore the values.
  # The following are filtered out of the restore env because they could be different from the checkpoint
  # values.  The JVM only allows new values to be set into the restored environment:
  # - INVOKED - this gets set to the path used to launch the server script;
  # It is not used for restore and we don't care if this changes.
  # Unsetting it is faster than filtering it out
  unset INVOKED
  # REGEX explanation
  # ^[...] is to negate the match for a specific char
  # [^\=] matches an '=' at the beginning of the line
  # *.= matches any '=' in the rest of the line.
  # We need to filter out lines that start with '=' in case a multi-line output starts with '='
  env | grep -E "^[^\=]*.=" > ${SERVER_OUTPUT_DIR}/workarea/checkpoint/.env.properties

  rm -f ${SERVER_OUTPUT_DIR}/workarea/checkpoint/.restoreFailedMarker
  
  criuRestore $BACKGROUND_RESTORE
  rc=$?
  if [ -f "${SERVER_OUTPUT_DIR}/workarea/checkpoint/.restoreFailedMarker" ]; then
    rc=`cat ${SERVER_OUTPUT_DIR}/workarea/checkpoint/.restoreFailedMarker`
    return $rc
  fi
  if [ $rc -ne 0 ]; then
    if [ "${CRIU_RESTORE_DISABLE_RECOVERY}" = "true" ]; then
      issueMessage --message:error.checkpoint.restore.failed.no.recovery "${X_LOG_DIR}/checkpoint/restore.log"
    else
      issueMessage --message:info.checkpoint.restore.failed "${X_LOG_DIR}/checkpoint/restore.log"
    fi

    if [ -f "${X_LOG_DIR}/checkpoint/restore.log" ]; then
      cat "${X_LOG_DIR}/checkpoint/restore.log"
    fi
  fi
  return $rc
}

JAVA_DEBUG=
JVM_OPTIONS_QUOTED=
INVOKED="$0"
export INVOKED

case "$ACTION" in

  # Start the server in the foreground
  run | debug | checkpoint)

    if serverEnvAndJVMOptions
    then
      :
    else
      exit $?
    fi

    if [ -d "${SERVER_OUTPUT_DIR}/workarea/checkpoint/image" ] && [ ${ACTION} != "checkpoint" ]; then
      # Avoid call to restoreServer below if server already running.
      # If there is no checkpoint, then maintain legacy behavior of 
      # allowing server launch to check for already running instance.
      if serverStatus false --status:starting; then
        exit 0
      fi  
    fi

    serverWorkingDirectory

    restoreServer false "$@"
    rc=$?
    if [ $rc = 0 ]; then
      exit 0
    # 104 is the code for no image found to restore (or --clean is used)
    elif [ $rc -ne 104 ]; then
      if [ "${CRIU_RESTORE_DISABLE_RECOVERY}" = "true" ]; then
        exit $rc
      fi
      # will try to recover with a normal launch below
    fi

    if checkServer true
    then
      if [ "${ACTION}" = "debug" ]
      then
        if [ -z "${WLP_DEBUG_ADDRESS}" ]
        then
          WLP_DEBUG_ADDRESS=7777
        fi
        if [ -z "${WLP_DEBUG_SUSPEND}" ]
        then
          WLP_DEBUG_SUSPEND=y
        fi
        if [ "${WLP_DEBUG_REMOTE}" = "y" ]
        then
          WLP_DEBUG_REMOTE_HOST="0.0.0.0:"
        else
          WLP_DEBUG_REMOTE_HOST=""
        fi
        JAVA_DEBUG="-Dwas.debug.mode=true -Dsun.reflect.noInflation=true -agentlib:jdwp=transport=dt_socket,server=y,suspend=${WLP_DEBUG_SUSPEND},address=${WLP_DEBUG_REMOTE_HOST}${WLP_DEBUG_ADDRESS}"
      fi

      # WLP_JAR_CYGWIN is used only in support of java -jar startup of a Liberty server.
      # When WLP_JAR_CYGWIN=true (or synonym) is specified, the PID of the launching 
      # process (i.e. parent of server process) is determined and stored in the PID file
      # for use by the java -jar shutdown hook to ensure termination of the launching 
      # process. This is necessary because the launching process does not terminate on
      # its own under cygwin and if left behind holds file locks that prevent cleanup
      # of the java -jar extraction directory. 
      if [ "$WLP_JAR_CYGWIN" = "true" -o "$WLP_JAR_CYGWIN" = "1" -o "$WLP_JAR_CYGWIN" = "TRUE" -o "$WLP_JAR_CYGWIN" = "True" ]
      then 
	      serverCmd '' "$@" &
	      PID=$!
	      mkdirs "${X_PID_DIR}"
	      # find parent PID and store it 
	      SPID=`ps -ef | while read l; do grep $PID | awk -v PID=$PID '{ if ( $3 == PID ) print $2 }'; done`
	      safeEcho "${SPID}" > "${X_PID_FILE}"
	      wait "${PID}" # wait because this 
	      rc=$?
      else
	      serverCmd '' "$@"
	      rc=$?
      fi
      exit $rc
    else
      exit 2
    fi
  ;;
   
  # Start the server in the background
  start)    

    if serverEnvAndJVMOptions
    then
      :
    else
      exit $?
    fi

    if checkServer true
    then
      # We can't start or restore the server in the background if it is already running.
      serverStatus false --status:starting
      rc=$?
      case $rc in
      "0" )
        rc=1
        ;;
      "1" )
        serverWorkingDirectory

        restoreServer true "$@"
        rc=$?
        if [ $rc = 0 ]; then
          exit 0
        # 104 is the code for no image found to restore (or --clean is used)
        elif [ $rc -ne 104 ]; then
          if [ "${CRIU_RESTORE_DISABLE_RECOVERY}" = "true" ]; then
            exit $rc
          fi
          # will try to recover with a normal launch below
        fi

        mkdirs "${X_LOG_DIR}"
        if [ -f "${X_LOG_DIR}/${X_LOG_FILE}" ] && [ ! -w "${X_LOG_DIR}/${X_LOG_FILE}" ]
        then
          javaCmd '' --message:error.fileNotFound "${X_LOG_DIR}/${X_LOG_FILE}"
          rc=$?
          if [ $rc = 0 ]; then
            rc=22
          fi
          exit $rc
        fi

        mkdirs "${X_PID_DIR}"
        rmIfExist "${X_PID_FILE}"

        mkdirs "${SERVER_OUTPUT_DIR}/workarea"
        touchIfNotExist "${SERVER_OUTPUT_DIR}/workarea/.sLock"
        rmIfExist "${SERVER_OUTPUT_DIR}/workarea/.sCommand"

        # Start the target server with the provided arguments
        serverCmd "${X_LOG_DIR}/${X_LOG_FILE}" "$@"
        rc=$?
        if [ $rc = 1 ]; then
          # The launcher failed (due to bad JVM arguments, for example).
          # Translate to a better error code.
          rc=22
        fi
        ;;
      esac
      exit $rc
    else
      exit 2
    fi    
  ;;

  create)
    installEnv
    createServer "$@"
    exit $rc
  ;;
  
  # Stop the server. 
  stop)
    serverEnv
    if checkServer true
    then
      PID=
      if [ -f "${X_PID_FILE}" ]; then
        PID=`cat "${X_PID_FILE}"`
      fi

      javaCmd '' "${SERVER_NAME}" --pid="${PID}" --stop "$@"
      rc=$?

      case "$rc" in
        "0" )
          rm -f "${X_PID_FILE}"
          ;;

        "1" )
          rmIfExist "${X_PID_FILE}" "${SERVER_OUTPUT_DIR}/workarea/.sCommand"
          ;;
      esac
      exit $rc
    else
      exit 2
    fi
  ;;
  
  # Check if server is running
  status)
    serverEnv
    if checkServer true
    then
      serverPIDStatus
      mkdirs "${X_LOG_DIR}"
      javaCmd '' "${SERVER_NAME}" --pid="${PID}" --status
      rc=$?

      if [ $rc = 1 ]; then
        rmIfExist "${X_PID_FILE}" "${SERVER_OUTPUT_DIR}/workarea/.sCommand"
      fi
      exit $rc
    else
      exit 2
    fi
  ;;

  status:fast)
    serverEnv
    if checkServer false
    then
      serverStatus true --status
      exit $?
    else
      exit 2
    fi
  ;;

  # Package up the runtime and target server configuration
  package)
    if serverEnvAndJVMOptions
    then
      :
    else
     exit $?
    fi
    
    # Check to see if the server exists
    if checkServer true
    then
      serverPIDStatus
      JVM_OPTIONS_QUOTED=${SERVER_JVM_OPTIONS_QUOTED}
      javaCmd '' "${JVM_OPTIONS_QUOTED}" "${SERVER_NAME}" --pid="${PID}" --package "$@" 
      exit $?
    else
      exit 2
    fi    
  ;;
  

  # dump the server's configs/logs/status to a zip file
  dump)
    serverEnv
    # Check to see if the server exists
    if checkServer true
    then
      javaCmd '' "${SERVER_NAME}" --dump "$@" 
      exit $?
    else
      exit 2
    fi    
  ;;

  # dump JVM status
  javadump)
    serverEnv
    if checkServer true
    then
      javaCmd '' "${SERVER_NAME}" --javadump "$@"
      exit $?
    else
      exit 2
    fi
  ;;
  
  # pause the server
  pause)
    serverEnv
    # Check to see if the server exists
    if checkServer true
    then
      javaCmd '' "${SERVER_NAME}" --pause "$@" 
      exit $?
    else
      exit 2
    fi    
  ;;

  # resume the server
  resume)
    serverEnv
    # Check to see if the server exists
    if checkServer true
    then
      javaCmd '' "${SERVER_NAME}" --resume "$@" 
      exit $?
    else
      exit 2
    fi    
  ;;

  version)
    serverEnv
    javaCmd '' --version
  ;;  
  
  list)
    installEnv
    javaCmd '' --list
  ;;  
  
  help)
    installEnv
    javaCmd '' --help ${SERVER_NAME}
  ;;  

  help:usage)
    installEnv
    javaCmd '' --help:usage
  ;;
  
  *)
    installEnv
    javaCmd '' --help:actions:${ACTION}
  ;;
esac

exit 0
