/*
 * Copyright (c) Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2018 All Rights Reserved
 *
 * Licensed under CLOUD LINUX LICENSE AGREEMENT
 * http://cloudlinux.com/docs/LICENSE.TXT
 *
 * The part the support library for lsapi & proxy_lsapi Apache modules
 * author Alexander Demeshko <ademeshko@cloudlinux.com>
 *
 */

#include <lscapi_util.h>

#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <fcntl.h>
#include <ctype.h>

#include <unixd.h>
#include <mpm_common.h>
#include <ap_mpm.h>
#include <apr_thread_proc.h>
#include <apr_strings.h>
#include <apr_queue.h>
#include <apr_global_mutex.h>
#include <apr_version.h>
#include <apr_hash.h>
#if APR_MAJOR_VERSION < 2
#include <apr_support.h>
#endif
#include <http_config.h>
#include <util_time.h>

#if MODULE_MAGIC_NUMBER_MAJOR >= 20090209
#include "mod_unixd.h"
#endif

#if MODULE_MAGIC_NUMBER_MAJOR < 20081201
#define ap_unixd_config unixd_config
#define ap_unixd_setup_child unixd_setup_child
#endif


apr_status_t lscapi_mutex_pthread_create(apr_global_mutex_t **mutex,
                                         const char **lockfile,
                                         const char *mutex_type,
                                         apr_pool_t *pool,
                                         server_rec *main_server,
                                         lsapi_svr_conf_t *cfg);


pid_t lscapi_spawn_backend_lsphp(server_rec *main_server,
                                 spawn_info_t *spawn_info,
                                 int restore,
                                 int log_file);

struct lsapi_starter_command {
    spawn_info_t spawn_info;
    int force_start;
    server_rec *req_server;
};
typedef struct lsapi_starter_command lsapi_starter_command;

struct lsapi_starter_notify {
    pid_t pid;
    int err;
};
typedef struct lsapi_starter_notify lsapi_starter_notify;

/* The APR other-child API doesn't tell us how the daemon exited
 * (SIGSEGV vs. exit(1)).  The other-child maintenance function
 * needs to decide whether to restart the daemon after a failure
 * based on whether or not it exited due to a fatal startup error
 * or something that happened at steady-state.  This exit status
 * is unlikely to collide with exit signals.
 */
#define DAEMON_STARTUP_ERROR 254


static int g_wakeup_timeout = 1;
static int g_wakeup_counter = 10;
static apr_proc_t *g_starter_proc = NULL;
static apr_file_t *g_st_read_pipe = NULL;
static apr_file_t *g_st_write_pipe = NULL;
static apr_file_t *g_ap_write_pipe = NULL;
static apr_file_t *g_ap_read_pipe = NULL;
static apr_global_mutex_t *g_pipelock = NULL;
static const char *g_pipelock_name;
static const char *g_pipelock_mutex_type = "lsapi-pipe";
static const char *g_prefix = "lsapi-util: ";

static int volatile g_caughtSigTerm = 0;
static pid_t g_starter_pid = 0;

static int use_starter_hash_t_structures = 0;

void set_use_starter_hash_t_structures(void)
{
    use_starter_hash_t_structures = 1;
}


int lsapi_is_server_restart(void)
{
    int rc, mpm_state;

    rc = ap_mpm_query(AP_MPMQ_MPM_STATE, &mpm_state);

    /*
        We expect that on restart (both immediate and graceful) mpm_query hook is not registered
        and ap_mpm_query will return APR_EGENERAL. (it doesn't work as of 06.11.2018)
    */
    if (rc == APR_EGENERAL)
        return rc;

    if(mpm_state != AP_MPMQ_RUNNING)
        return APR_EOF;

    return APR_SUCCESS;
}

static int is_threaded;
/* LSAPI_PRO_ONLY:BEGIN */
static int mutex_tweak;
/* LSAPI_PRO_ONLY:END */

static apr_status_t global_cleanup(void *dummy)
{
    /*
      In fact now starter ignores all configs about mutex mech 
             and creates g_pipelock as pthreaded mutex in starter_init
      So we can safely do not destroy it in case of threaded MPM?
    */
/* LSAPI_PRO_ONLY:BEGIN */
    /*
      But it's still possible to force mutex destroy by adding into lsapi config the following line:
      lsapi_mutex_tweak On
    */
/* LSAPI_PRO_ONLY:END */
    if(g_pipelock != NULL)
    {
        if(!is_threaded 
/* LSAPI_PRO_ONLY:BEGIN */
           || mutex_tweak
/* LSAPI_PRO_ONLY:END */
        )
        {
            apr_global_mutex_destroy(g_pipelock);
            g_pipelock = NULL;
        }
    }

    return APR_SUCCESS;
}

static void signal_handler(int signo)
{
    /* Sanity check, Make sure I am not the subprocess. A subprocess may
       get signal after fork() and before execve() */
    if (getpid() != g_starter_pid) {
        exit(0);
        return;
    }

    if (   (signo == SIGTERM)
        || (signo == SIGUSR1)
        || (signo == SIGHUP)
#ifdef AP_SIG_GRACEFUL_STOP
        || (signo == AP_SIG_GRACEFUL_STOP)
#endif
    ) {
        g_caughtSigTerm = 1;
    }
}

static inline int starter_should_exit(void)
{
    return g_caughtSigTerm;
}

static apr_status_t init_signal(const char *prefix, server_rec * main_server)
{
    struct sigaction sa;

    /* Setup handlers */
    sa.sa_handler = signal_handler;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = 0;

    if (sigaction(SIGTERM, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install SIGTERM handler", prefix);
        return APR_EGENERAL;
    }

    /* Httpd restart */
    if (sigaction(SIGHUP, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install SIGHUP handler", prefix);
        return APR_EGENERAL;
    }

    /* Httpd graceful restart */
    if (sigaction(SIGUSR1, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install SIGUSR1 handler", prefix);
        return APR_EGENERAL;
    }

    /* Httpd graceful stop */
#ifdef AP_SIG_GRACEFUL_STOP
    if (sigaction(AP_SIG_GRACEFUL_STOP, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install GRACEFUL_STOP handler", prefix);
        return APR_EGENERAL;
    }
#endif

    /* Ignore SIGPIPE */
    sa.sa_handler = SIG_IGN;
    if (sigaction(SIGPIPE, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install SIGPIPE handler", prefix);
        return APR_EGENERAL;
    }

    /* Ignore SIGCHLD */
    if (sigaction(SIGCHLD, &sa, NULL) < 0) {
        lscapi_log(APLOG_ERR, errno, main_server,
                     "%s Can't install SIGCHLD handler", prefix);
        return APR_EGENERAL;
    }

    return APR_SUCCESS;
}

static int set_group_privs(void)
{
    if (!geteuid()) {
        const char *name;

        /* Get username if passed as a uid */
        if (ap_unixd_config.user_name[0] == '#') {
            struct passwd *ent;

            uid_t uid = atoi(&ap_unixd_config.user_name[1]);

            if ((ent = getpwuid(uid)) == NULL) {
                lscapi_log(APLOG_ALERT, errno, NULL,
                             "getpwuid: couldn't determine user name from uid %"APR_PID_T_FMT", "
                             "you probably need to modify the User directive",
                             uid);
                return -1;
            }
            name = ent->pw_name;
        }

        else
            name = ap_unixd_config.user_name;

#if !defined(OS2) && !defined(TPF)
        /* OS/2 and TPF don't support groups. */

        /*
         * Set the GID before initgroups(), since on some platforms
         * setgid() is known to zap the group list.
         */
        if (setgid(ap_unixd_config.group_id) == -1) {
            lscapi_log(APLOG_ALERT, errno, NULL,
                         "setgid: unable to set group id to Group %"APR_PID_T_FMT,
                         ap_unixd_config.group_id);
            return -1;
        }

        /* Reset `groups' attributes. */
        if (initgroups(name, ap_unixd_config.group_id) == -1) {
            lscapi_log(APLOG_ALERT, errno, NULL,
                         "initgroups: unable to set groups for User %s "
                         "and Group %"APR_PID_T_FMT, name,
                         ap_unixd_config.group_id);
            return -1;
        }
#endif                          /* !defined(OS2) && !defined(TPF) */
    }
    return 0;
}


static void starter_maint(int reason, void *data, apr_wait_t status)
{
    apr_proc_t *proc = data;
    int mpm_state;

    switch (reason) {
    case APR_OC_REASON_DEATH:
        apr_proc_other_child_unregister(data);
        if (ap_mpm_query(AP_MPMQ_MPM_STATE, &mpm_state) == APR_SUCCESS
            && mpm_state != AP_MPMQ_STOPPING) {
            if (status == DAEMON_STARTUP_ERROR) {
                lscapi_log(APLOG_CRIT, 0, NULL,
                             "%s lsapi starter failed to initialize; stopping httpd", g_prefix);
                /*
                 * try to terminate httpd
                 */
                kill(getpid(), SIGTERM);

            }
            else {
                lscapi_log(APLOG_ERR, 0, NULL,
                             "%s lsapi starter died, restarting the server", g_prefix);

                /* HACK: I can't just call create_process_manager() to
                   restart a process manager, because it will use the dirty
                   share memory, I have to kill myself a SIGHUP, to make
                   a clean restart */
                /* FIXME: This is the httpd parent; it is doing a hard
                 * restart of the server!
                 */
                if (kill(getpid(), SIGHUP) < 0) {
                    lscapi_log(APLOG_EMERG,
                                 apr_get_os_error(), NULL,
                                 "%s can't send SIGHUP to self", g_prefix);
                    exit(0);
                }
            }
        }
        break;
    case APR_OC_REASON_RESTART:
        apr_proc_other_child_unregister(data);
        break;
    case APR_OC_REASON_LOST:
        apr_proc_other_child_unregister(data);
        /* It hack here too, a note above */
        /* FIXME: This is the httpd parent; mod_lsapi is doing a hard
         * restart of the server!
         */
        if (kill(getpid(), SIGHUP) < 0) {
            lscapi_log(APLOG_EMERG,
                         apr_get_os_error(), NULL,
                         "%s can't send SIGHUP to self", g_prefix);
            exit(0);
        }
        break;
    case APR_OC_REASON_UNREGISTER:
        /* I don't think it's going to happen */
        kill(proc->pid, SIGHUP);
        break;
    }
}

static apr_status_t starter_main(const char *prefix, server_rec * main_server, apr_pool_t * config_pool);

apr_status_t lscapi_starter_child_init(server_rec* main_server, apr_pool_t* config_pool, const char *prefix)
{
    apr_status_t rc;

    if ((rc = apr_global_mutex_child_init(&g_pipelock, g_pipelock_name,
                                          main_server->process->pool)) != APR_SUCCESS) {
        lscapi_log(APLOG_EMERG, rc, main_server,
                     "%s apr_global_mutex_child_init error for pipe mutex", prefix);
        exit(1);
    }

    return APR_SUCCESS;
}

apr_status_t lscapi_starter_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp)
{
    return lscapi_mutex_register(g_pipelock_mutex_type, p);
}

static int g_logfile = -1;

int starter_log_error(server_rec *s, int errnum, const char *fmt, ...) {
    if(g_logfile < 0) return -1;

    char errbuf[MAX_STRING_LEN];
    va_list args;
    int errlen;
    int len = 0;

    errbuf[len++] = '[';
#ifdef APACHE2_4
    errlen = sizeof errbuf - len;
    ap_recent_ctime_ex(errbuf + len, apr_time_now(), AP_CTIME_OPTION_NONE, &errlen);
    len += errlen-1;
#else
    ap_recent_ctime(errbuf + len, apr_time_now());
    errlen = strlen(errbuf + len);
    len += errlen;
#endif
    errbuf[len++] = ']';
    errbuf[len++] = ' ';

    if(errnum != 0) {
        errlen = sizeof errbuf - len;
        if(errnum > 0) {
            errlen = snprintf(errbuf+len, errlen, "[ERROR %s:%d] ", strerror(errnum), errnum);
        } else {
            errlen = snprintf(errbuf+len, errlen, "[ERROR] ");
        }
        len += errlen;
    }

    va_start(args, fmt);
    errlen = sizeof errbuf - len - 1; // one less for trailing \n
    errlen = apr_vsnprintf(errbuf+len, errlen, fmt, args);
    len += errlen;
    va_end(args);

    errbuf[len++] = '\n';

    return write(g_logfile, errbuf, len);
}

static apr_status_t make_dir(server_rec *main_server, apr_pool_t *config_pool,
                             const char *socket_path, const char *socket_descr)
{
    apr_status_t rc;
    apr_finfo_t finfo;

    rc = apr_stat(&finfo, socket_path, APR_FINFO_USER, config_pool);
    if (rc != APR_SUCCESS) {

        if ((rc = apr_dir_make_recursive(socket_path,
                                         APR_UREAD | APR_UWRITE | APR_UEXECUTE,
                                         config_pool)) != APR_SUCCESS) {
            lscapi_log(APLOG_ERR, rc, main_server,
                         "%s Can't create %s %s",
                         g_prefix, socket_descr, socket_path);
            return APR_ENOSTAT;
        }

        if (!geteuid()) {
            if (chown(socket_path,
                      ap_unixd_config.user_id, ap_unixd_config.group_id) < 0) {
                lscapi_log(APLOG_ERR, errno, main_server,
                             "%s Can't set ownership of %s %s",
                             g_prefix, socket_descr, socket_path);
                return APR_EGENERAL;
            }
        } //if (!geteuid())

    } //if (rc != APR_SUCCESS)

    return APR_SUCCESS;
}

apr_status_t lscapi_starter_init(const char *prefix, server_rec *main_server, apr_pool_t *config_pool)
{
    apr_status_t rc;
    lsapi_svr_conf_t *main_config = lsapi_get_svr_config(main_server);

    g_prefix = prefix;

    /* Make dir for unix domain socket */
    rc = make_dir(main_server, config_pool, main_config->selfstarter_socket_path, "selfstarter socket dir");
    if (rc != APR_SUCCESS) {
        exit(1);
    }

/* LSAPI_PRO_ONLY:BEGIN */
#ifdef WITH_CRIU
#if 0
/*
   MODLS-623
   TODO: This call to make_dir partially breaks config_pool for unknown reason,
   so Apache mutex-mech semaphores leaves after Apache restart. 
*/
    /* Make dir for criu images */
    // Default value is still not applied at this moment
    const char *criu_imgs_dir_path = cfg->criu_imgs_dir_path ? cfg->criu_imgs_dir_path : (cfg->socket_path ? cfg->socket_path : LSAPI_SOCK_PATH);
    make_dir(main_server, config_pool, criu_imgs_dir_path, "criu imgs dir");
#endif
#endif
/* LSAPI_PRO_ONLY:END */

    /* Create mutex for pipe reading and writing */
    rc = lscapi_mutex_pthread_create(&g_pipelock, &g_pipelock_name, g_pipelock_mutex_type,
                                   main_server->process->pool, main_server, main_config);
    if (rc != APR_SUCCESS) {
        exit(1);
    }

    ap_mpm_query(AP_MPMQ_IS_THREADED, &is_threaded);
/* LSAPI_PRO_ONLY:BEGIN */
    mutex_tweak = main_config->mutex_tweak;
/* LSAPI_PRO_ONLY:END */

    apr_pool_cleanup_register(config_pool, NULL, global_cleanup, apr_pool_cleanup_null);

    /* Create pipes to communicate between starter and apache */
    if ((rc = apr_file_pipe_create_ex(&g_st_read_pipe, &g_ap_write_pipe,
                                      APR_FULL_BLOCK,  config_pool)) != APR_SUCCESS
        || (rc = apr_file_pipe_create_ex(&g_ap_read_pipe, &g_st_write_pipe,
                                      APR_FULL_BLOCK, config_pool))) {
        lscapi_log(APLOG_ERR, rc, main_server,
                     "%s Can't create pipe between starter and module", prefix);
        return rc;
    }
    apr_file_pipe_timeout_set(g_ap_write_pipe, apr_time_from_sec(g_wakeup_timeout));
    apr_file_pipe_timeout_set(g_st_write_pipe, apr_time_from_sec(g_wakeup_timeout));
    apr_file_pipe_timeout_set(g_ap_read_pipe, apr_time_from_sec(g_wakeup_timeout));
    apr_file_pipe_timeout_set(g_st_read_pipe, apr_time_from_sec(g_wakeup_timeout));

    /* Spawn the starter */
    g_starter_proc =
        (apr_proc_t *) apr_pcalloc(config_pool, sizeof(*g_starter_proc));
    rc = apr_proc_fork(g_starter_proc, config_pool);
    if (rc == APR_INCHILD) {
        /* I am the child */
        g_starter_pid = getpid();
        lscapi_log(APLOG_NOTICE, 0, main_server,
                     "%s Selfstarter %" APR_PID_T_FMT  " started", prefix, getpid());

        if ((rc = init_signal(prefix, main_server)) != APR_SUCCESS) {
            lscapi_log(APLOG_EMERG, rc, main_server,
                         "%s can't install signal handler, exiting now", prefix);
            exit(DAEMON_STARTUP_ERROR);
        }

        if (getuid() != 0) {
            lscapi_log(APLOG_EMERG, 0, main_server,
                            "%s current user is not root, exiting now", prefix);
            exit(DAEMON_STARTUP_ERROR);
        }

        if (main_config->sulsphp_log_path != SULSPHP_LOG_UNINITIALIZED)
        {
            char *log_path = main_config->sulsphp_log_path;
            size_t len = strlen(log_path);
            size_t full_len = len + (log_path[len - 1] == '/' ? 0 : 1) + strlen(SULSPHP_LOG_NAME);
            if (full_len < PATH_MAX)
            {
                char sulsphp_log_path[PATH_MAX];
                snprintf(sulsphp_log_path, PATH_MAX, "%s%s%s",
                         log_path, log_path[len - 1] == '/' ? "" : "/", SULSPHP_LOG_NAME);
                g_logfile = open(sulsphp_log_path, O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE, 0644);
                if (g_logfile < 0)
                {
                    lscapi_log(APLOG_WARNING, errno, main_server,
                               "Can't open sulsphp log file %s: %s. Using the default path",
                               sulsphp_log_path, strerror(errno));
                }
            }
            else
            {
                lscapi_log(APLOG_WARNING, 0, main_server,
                           "Length of the sulsphp log file path is too long (%zu). Max length (%zu). Using the default path",
                           full_len, (size_t)PATH_MAX - 1);
            }
        }
        if (g_logfile < 0)
        {
            const char *logfile_name_default = lscapi_make_fname_in_logdir(main_server, config_pool, SULSPHP_LOG_NAME);
            g_logfile = open(logfile_name_default, O_WRONLY | O_APPEND | O_CREAT | O_LARGEFILE, 0644);
            if (g_logfile < 0)
            {
                lscapi_log(APLOG_WARNING, errno, main_server,
                           "Can't open default sulsphp log file %s: %s.", logfile_name_default, strerror(errno));
            }
        }

        if (set_group_privs()) {
            lscapi_log(APLOG_EMERG, errno, main_server,
                            "%s Can't set group privs, exiting now", prefix);
            exit(DAEMON_STARTUP_ERROR);
        }


        if (NULL != ap_unixd_config.chroot_dir) {
            if (chdir(ap_unixd_config.chroot_dir) != 0) {
                lscapi_log(APLOG_EMERG, errno, main_server,
                            "%s Can't chdir to %s, exiting now", prefix, ap_unixd_config.chroot_dir);
                exit(DAEMON_STARTUP_ERROR);
            }
            if (chroot(ap_unixd_config.chroot_dir) != 0) {
                lscapi_log(APLOG_EMERG, errno, main_server,
                            "%s Can't chroot to %s, exiting now", prefix, ap_unixd_config.chroot_dir);
                exit(DAEMON_STARTUP_ERROR);
            }
            if (chdir("/") != 0) {
                lscapi_log(APLOG_EMERG, errno, main_server,
                            "%s Can't chdir to new root, exiting now", prefix);
                exit(DAEMON_STARTUP_ERROR);
            }
        }

        // Starter will (in lscapi_spawn_lsphp) change its privs several times up and down.
        // So it should have superuser privs in its saved set-user-ID
        // ruid will be nobody 
        if (setresuid(ap_unixd_config.user_id, ap_unixd_config.user_id, 0) == -1) {
            lscapi_log(APLOG_EMERG, errno, main_server,
                        "%s unable to change uid to %ld, exiting now", prefix, (long) ap_unixd_config.user_id);
            exit(DAEMON_STARTUP_ERROR);
        }

#if defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
        /* this applies to Linux 2.4+ */
        if (ap_coredumpdir_configured) {
            if (prctl(PR_SET_DUMPABLE, 1)) {
                lscapi_log(APLOG_EMERG, errno, main_server,
                            "%s Set dumpable failed - this child will not coredump"
                            " after software errors", prefix );
            }
        }
#endif

        apr_file_close(g_ap_write_pipe);
        apr_file_close(g_ap_read_pipe);

        starter_main(prefix, main_server, config_pool);

        lscapi_log(APLOG_NOTICE, 0, main_server,
                    "%s Selfstarter %" APR_PID_T_FMT " stopped", prefix, getpid());
        exit(0);

    } else if (rc != APR_INPARENT) {
        lscapi_log(APLOG_EMERG, rc, main_server,
                    "%s Create selfstarter error", prefix );
        exit(1);
    }

    /* I am the parent
        I will send the stop signal in procmgr_stop_procmgr() */
    apr_pool_note_subprocess(config_pool, g_starter_proc,
                                APR_KILL_ONLY_ONCE);
    apr_proc_other_child_register(g_starter_proc, starter_maint,
                                  g_starter_proc, NULL, config_pool);

    return APR_SUCCESS;
}


static apr_status_t wait_for_io_or_timeout(apr_file_t *f, int for_read, int ignore_restart)
{
    int i;
    apr_status_t rc;

    for (i = 0; i < g_wakeup_counter; i++)
    {
        if (ignore_restart == 0 && starter_should_exit())
        {
            // Use EINIT to indicate Apache restart status
            return APR_EINIT;
        }
        rc = apr_wait_for_io_or_timeout(f, NULL, for_read);

        if (APR_STATUS_IS_TIMEUP(rc))
        {
            continue;
        }

        /* Both APR_SUCCESS and any error */
        return rc;
    }

    return APR_TIMEUP;
}

static apr_status_t starter_fetch_cmd(const char* prefix,
                                      lsapi_starter_command *command,
                                      server_rec* main_server,
                                      apr_pool_t *config_pool,
                                      apr_hash_t *hash_table)
{
    apr_status_t rc;
    const char *errmsg = NULL;

    lsapi_svr_conf_t *cfg = lsapi_get_svr_config(main_server);

    /* Sanity check */
    if (!g_st_read_pipe)
    {
        lscapi_log(APLOG_WARNING, 0, main_server,
                   "%s STARTER: client pipe is NOT ready", prefix);
        return APR_EPIPE;
    }

    /* Wait for next command */
    lscapi_notice(cfg, 0, main_server, "%s STARTER: just before waiting for spawnCmd on starter pipe", prefix);
    rc = wait_for_io_or_timeout(g_st_read_pipe, 1, 0);
    if (rc == APR_EINIT)
    {
        lscapi_notice(cfg, 0, main_server, "%s STARTER: restart condition found on waiting for spawnCmd", prefix);
        return APR_EINIT;
    }

    /* Timeout */
    if (APR_STATUS_IS_TIMEUP(rc))
    {
        lscapi_notice(cfg, 0, main_server, "%s STARTER: timeout exceeded on waiting for spawnCmd", prefix);
        return APR_TIMEUP;
    }
    /* Something wrong */
    if (rc != APR_SUCCESS) {
        errmsg = "waiting for spawnCmd read";
        goto exit;
    }
    /* rc == APR_SUCCESS */
    if (!use_starter_hash_t_structures) {

        rc = apr_file_read_full(g_st_read_pipe, command, sizeof(*command), NULL);
        if (rc != APR_SUCCESS) {
            errmsg = "reading of spawnCmd";
            goto exit;
        }

        lscapi_notice(cfg, 0, main_server,"%s STARTER: spawnCmd received", prefix);

        return rc;
    }

    int is_hashed = 0;
    char hostname_key[256];
    apr_size_t msgsize;
    msgsize = sizeof(hostname_key);

    rc = apr_file_read(g_st_read_pipe, &hostname_key, &msgsize);
    if (rc != APR_SUCCESS) {
        errmsg = "reading of hostname";
        goto exit;
    }

    lscapi_notice(cfg, 0, main_server, "%s STARTER hostname received: %s", prefix, hostname_key);
    lsapi_starter_command *command_ptr = apr_hash_get(hash_table, hostname_key, APR_HASH_KEY_STRING);

    if (!command_ptr) {

        is_hashed = 0;
        msgsize = sizeof(is_hashed);

        lscapi_notice(cfg, rc, main_server,
                      "%s STARTER: not found hash in table for: %s", prefix, hostname_key);

        // we should finish command processing, even despite of restart condition
        rc = wait_for_io_or_timeout(g_st_write_pipe, 0, 1);
        /* Timeout */
        if (APR_STATUS_IS_TIMEUP(rc))
        {
            lscapi_notice(cfg, 0, main_server, "%s STARTER: timeout exceeded on waiting for response write", prefix);
            return APR_TIMEUP;
        }
        /* Something wrong */
        if (rc != APR_SUCCESS) {
            errmsg = "waiting for response write";
            goto exit;
        }
        /* rc == APR_SUCCESS */
        rc = apr_file_write_full(g_st_write_pipe, &is_hashed, msgsize, NULL);
        if (rc != APR_SUCCESS) {
            errmsg = "writing isHashed=0 flag";
            goto exit;
        }

        lscapi_notice(cfg, rc, main_server,
                      "%s STARTER send not found to client: %s", prefix, hostname_key);

        // we should finish command processing, even despite of restart condition
        rc = wait_for_io_or_timeout(g_st_read_pipe, 1, 1);
        /* Timeout */
        if (APR_STATUS_IS_TIMEUP(rc))
        {
            lscapi_notice(cfg, 0, main_server, "%s STARTER: timeout exceeded on waiting for hashed spawnCmd read", prefix);
            return APR_TIMEUP;
        }
        /* Something wrong */
        if (rc != APR_SUCCESS) {
            errmsg = "waiting for hashed spawnCmd read";
            goto exit;
        }
        /* rc == APR_SUCCESS */
        rc = apr_file_read_full(g_st_read_pipe, command, sizeof(lsapi_starter_command), NULL);
        if (rc != APR_SUCCESS) {
            errmsg = "reading hashed spawnCmd";
            goto exit;
        }

        lscapi_notice(cfg, rc, main_server,
                      "%s STARTER: command received for: %s", prefix, hostname_key);

        command_ptr = apr_palloc(config_pool, sizeof(lsapi_starter_command));

        *command_ptr = *command;

        apr_hash_set(hash_table, apr_pstrdup(config_pool,hostname_key), APR_HASH_KEY_STRING, command_ptr);

        lscapi_notice(cfg, rc, main_server,
                      "%s STARTER: info to hash table is written for: %s", prefix, hostname_key);

        unsigned int hash_t_size = apr_hash_count(hash_table) * sizeof(lsapi_starter_command);

        lscapi_notice(cfg, rc, main_server,
                      "%s STARTER: result size of storage : %i bytes", prefix, hash_t_size);
        return rc;
    }

    is_hashed = 1;
    msgsize = sizeof(is_hashed);

    lscapi_notice(cfg, rc, main_server,
                  "%s STARTER: found hash in table for: %s", prefix, hostname_key);

    // we should finish command processing, even despite of restart condition
    rc = wait_for_io_or_timeout(g_st_write_pipe, 0, 1);
    /* Timeout */
    if (APR_STATUS_IS_TIMEUP(rc))
    {
        lscapi_notice(cfg, 0, main_server, "%s STARTER: timeout exceeded on waiting for response write", prefix);
        return APR_TIMEUP;
    }
    /* Something wrong */
    if (rc != APR_SUCCESS) {
        errmsg = "waiting for response write";
        goto exit;
    }
    /* rc == APR_SUCCESS */
    rc = apr_file_write_full(g_st_write_pipe, &is_hashed, msgsize, NULL);
    if (rc != APR_SUCCESS) {
        errmsg = "writing isHashed=1 flag";
        goto exit;
    }

    lscapi_notice(cfg, rc, main_server,
                  "%s STARTER: send found to client: %s", prefix, hostname_key);

    *command = *command_ptr;

exit:
    if(errmsg != NULL) {
        if(starter_should_exit()) {
            lscapi_notice(cfg, 0, main_server,"%s STARTER: %s: interrupted due to server restart", prefix, errmsg);
        } else {
            lscapi_notice(cfg, 0, main_server, "%s STARTER: %d: on %s", prefix, rc, errmsg);
        }
    }
    return rc;
}

static void starter_spawn(const char *prefix,
                          lsapi_starter_command *command,
                          lsapi_starter_notify *notify,
                          server_rec *main_server,
                          apr_pool_t *config_pool)
{

    pid_t pid;
    server_rec *workserver = command->req_server ? command->req_server : main_server;

/* LSAPI_PRO_ONLY:BEGIN */
#ifdef WITH_CRIU
    /*
     * Try restore from images first.
     * pid_t lscapi_spawn_backend(spawn_info_t *spawn_info, int restore, int log_sock);
     * when restore > 0 it uses lscapi_restore() inside
     * which does just what we want -- it daemonizes a process
     * pid of that process are in return value.
     */
    if (command->spawn_info.use_criu) {
        pid = lscapi_spawn_backend_lsphp(main_server, &(command->spawn_info), 1, g_logfile);
        if (pid > 0) {
            starter_log_error(main_server, 0, "sulsphp: uid:(%u); gid:(%u); lveuid:(%u); cmd:(%s); CRIU: pid:(%d) restored from images",
                              command->spawn_info.real_uid, command->spawn_info.real_gid, command->spawn_info.lve_uid, command->spawn_info.cmd, pid);
            goto exit;
        } else if (pid == -ECONNREFUSED) {
            lscapi_log(APLOG_WARNING, 0, main_server, "%s Can't connect to CRIU. Please check that CRIU service is running.", prefix);
        } else if (pid != -11)
            starter_log_error(main_server, 0, "sulsphp: uid:(%u); gid:(%u); lveuid:(%u); cmd:(%s); CRIU: Failed restore from images - ignore",
                              command->spawn_info.real_uid, command->spawn_info.real_gid, command->spawn_info.lve_uid, command->spawn_info.cmd);
    }
#endif
/* LSAPI_PRO_ONLY:END */

    if ((pid = fork()) < 0) {
        notify->pid = -1;
        notify->err = errno;
        return;
    }
    else if (pid == 0) {
        /* child process */

        /* daemonize new process */
        setsid();
        setpgid(0, 0);

        if( command->spawn_info.backend_coredump ) {
            struct rlimit rlim;
            rlim.rlim_cur = RLIM_INFINITY;
            rlim.rlim_max = RLIM_INFINITY;
            setrlimit(RLIMIT_CORE, &rlim);
        }

        if (lscapi_spawn_backend_lsphp(workserver, &(command->spawn_info), 0, g_logfile) < 0)
            _exit(-1);
        _exit(0);
    }

/* LSAPI_PRO_ONLY:BEGIN */
#ifdef WITH_CRIU
exit:
#endif
/* LSAPI_PRO_ONLY:END */
    notify->pid = pid;
    notify->err = 0;
}

static apr_status_t starter_main(const char *prefix,
                                 server_rec * main_server,
                                 apr_pool_t * config_pool)
{
    lsapi_starter_command command;
    lsapi_starter_notify notify;
    apr_size_t notify_size;
    apr_status_t rc;
    lsapi_svr_conf_t *cfg = lsapi_get_svr_config(main_server);
    apr_hash_t *starttime_by_socket = apr_hash_make(config_pool);
    apr_hash_t *starter_hash_t = apr_hash_make(config_pool);

    if (use_starter_hash_t_structures) {
        lscapi_notice(cfg, 0, main_server,
                      "%s Starter use hashing structures against transmission over pipe ", prefix);
    } else {
        lscapi_notice(cfg, 0, main_server,
                      "%s Starter use transmission over pipe against hashing structures ", prefix);
    }

/* LSAPI_PRO_ONLY:BEGIN */
#ifndef WITHOUT_SENTRY
    lscapi_process_sentry_log();
#endif
/* LSAPI_PRO_ONLY:END */

    while(1) {
        if(starter_should_exit())
        {
            lscapi_notice(cfg, 0, main_server,
                          "%s Starter main loop: leave due to server restart", prefix);
            break;
        }

/* LSAPI_PRO_ONLY:BEGIN */
#ifdef WITH_CRIU
        lscapi_check_imgs_cleanup(cfg->criu_imgs_dir_path ? cfg->criu_imgs_dir_path : (cfg->socket_path ? cfg->socket_path : LSAPI_SOCK_PATH) );
#endif
/* LSAPI_PRO_ONLY:END */

        /* Wait for command */
        rc = starter_fetch_cmd(prefix, &command, main_server, config_pool, starter_hash_t);
        if (rc != APR_SUCCESS)
        {
            if(starter_should_exit())
            {
                lscapi_notice(cfg, 0, main_server,
                              "   %s Starter fetch failed: leave due to server restart", prefix);
                break;
            }
            //sleep(1);
            continue;
        }

        /* rc == APR_SUCCESS */

        // HACK. Starter knows that socket name is in argv[4]
        const char *key = command.spawn_info.socket_name;

        apr_time_t *startTimePtr = apr_hash_get(starttime_by_socket, key, APR_HASH_KEY_STRING);

        if (startTimePtr == NULL)
        {
            //lscapi_log(APLOG_DEBUG, 0, main_server, "Could not find starting time for (%s) - try to invoke", key);

            // the first invoking with this socket path
            //lscapi_log(APLOG_DEBUG, 0, main_server, "Try to invoke sulsphp %d", 1);
            starter_spawn(prefix, &command, &notify, main_server, config_pool);
            //lscapi_log(APLOG_DEBUG, 0, main_server, "After invoking sulsphp %d", 1);

            if(notify.pid > 0) {
                startTimePtr = apr_palloc(config_pool, sizeof(apr_time_t) );

                // store invoking time associating with socket path
                *startTimePtr = apr_time_now();
                //lscapi_log(APLOG_DEBUG, 0, main_server, "Setting starting time for (%s) to %ld", key, *startTimePtr);
                apr_hash_set(starttime_by_socket, key, APR_HASH_KEY_STRING, startTimePtr);
            //} else {
            //        lscapi_log(APLOG_DEBUG, 0, main_server, "Invoke (%s) successful so setting start time to %ld and storing it", key, *startTimePtr);
            }

        } else {

            //lscapi_log(APLOG_DEBUG, 0, main_server, "Starting time for (%s) is %ld", key, *startTimePtr);

#define STARTER_TIMEOUT  (2 * (APR_USEC_PER_SEC))

            if(command.force_start || apr_time_now() - *startTimePtr > STARTER_TIMEOUT) {

                //lscapi_log(APLOG_DEBUG, 0, main_server, "Try to invoke for(%s) due to force_start(%d) or to timediff(%ld) more than timeout(%ld)",
                //            key, command.force_start, apr_time_now() - *startTimePtr, STARTER_TIMEOUT);
                starter_spawn(prefix, &command, &notify, main_server, config_pool);
                //lscapi_log(APLOG_DEBUG, 0, main_server, "After invoking sulsphp %d", 2);

                if(notify.pid > 0) {
                    // store invoking time associating with socket path
                    *startTimePtr = apr_time_now();
                    //lscapi_log(APLOG_DEBUG, 0, main_server, "Invoke (%s) successful so setting start time to %ld",
                    //        key, *startTimePtr);
                //} else {
                    //lscapi_log(APLOG_DEBUG, 0, main_server, "Invoke (%s) failed so leaving start time prev %ld",
                    //        key, *startTimePtr);
                }

            } else {
                notify.pid = 0;  // special case of delayed invoke
                notify.err = 0;
            }

        }

        // we should finish the procedure, despite of restart
        rc = wait_for_io_or_timeout(g_st_write_pipe, 0, 1);
        if(APR_STATUS_IS_TIMEUP(rc))
        {
            lscapi_notice(cfg, 0, main_server, "%s Starter timeout exceeded on waiting for notify write", prefix);
            //sleep(1);
            continue;
        }
        if(rc != APR_SUCCESS)
        {
            if(starter_should_exit())
            {
                lscapi_notice(cfg, 0, main_server,
                              "   %s Starter before notify: leave due to server restart", prefix);
                break;
            }
            //sleep(1);
            continue;
        }

        /* rc == APR_SUCCESS */
        notify_size = sizeof notify;
        //lscapi_log(APLOG_NOTICE, 0, main_server,
        //                "%s: selfstarter just before write; pid:%d; err:%d", prefix, notify.pid, notify.err);
        if ((rc = apr_file_write(g_st_write_pipe, &notify, &notify_size)) != APR_SUCCESS) {
                lscapi_log(APLOG_WARNING, rc, main_server,
                            "%s can't send notify from selfstarter", prefix);
        } else {
            lscapi_notice(cfg, rc, main_server,
                          "%s STARTER notify sent to the client", prefix);
        }

    } //while (1)

    return APR_SUCCESS;
}

pid_t lscapi_starter_send_spawn_cmd(const spawn_info_t *spawn_info, request_rec *r, int force_start) {
    apr_status_t rc;
    int mpm_state;
    lsapi_starter_command command;
    lsapi_starter_notify notify;
    apr_size_t msgsize;
    const char *errptr = "";

    lsapi_svr_conf_t *cfg = lsapi_get_svr_config(r->server);

    command.spawn_info = *spawn_info;
    command.force_start = force_start;
    /*
        Here we send value of the pointer to another Apache process.
        But it is safe because all config are created only once, in the very first Apache process.
        And thus every Apache process has the same config pointer layout.
    */
    command.req_server = cfg->server_tweak ? r->server : NULL;


    /* Get the global mutex before posting the request */
    if ((rc = apr_global_mutex_lock(g_pipelock)) != APR_SUCCESS) {
        if (ap_mpm_query(AP_MPMQ_MPM_STATE, &mpm_state) == APR_SUCCESS
            && mpm_state == AP_MPMQ_STOPPING)
        {
                lscapi_rlog(APLOG_INFO, rc, r, "%s can't get pipe mutex due to Apache restart", g_prefix);
                return -1;
        }

        lscapi_rlog(APLOG_ERR, 0, r, "%s pipe mutex lock failed: %d", g_prefix, rc );
        return -1;
    }

    rc = wait_for_io_or_timeout(g_ap_write_pipe, 0, 0);
    if (rc == APR_EINIT)
    {
        errptr = ": Apache restart detected";
        goto exit;
    }
    if (APR_STATUS_IS_TIMEUP(rc))
    {
        errptr = ": timeout on waiting for writing into starter pipe is exceeded";
        goto exit;
    }
    if (rc != APR_SUCCESS)
    {
        errptr = ": error on waiting for writing into starter pipe";
        goto exit;
    }

    /* rc == APR_SUCCESS */
    if (use_starter_hash_t_structures == 0)
    {

        rc = apr_file_write_full(g_ap_write_pipe, &command, sizeof command, NULL);
        if (rc != APR_SUCCESS)
        {
             errptr = ": error on writing spawnCmd into starter pipe";
             goto exit;
        }

        lscapi_notice(cfg, rc, r->server,
                      "%s CLIENT: spawnCmd is sent to starter", g_prefix);
    } else
    {

        int is_hashed = 0;
        msgsize = strlen(r->hostname) * sizeof(char) + 1;

        rc = apr_file_write_full(g_ap_write_pipe, r->hostname, msgsize, NULL);
        if (rc != APR_SUCCESS)
        {
             errptr = ": error on writing hashed spawnCmd into starter pipe";
             goto exit;
        }

        lscapi_notice(cfg, rc, r->server,
                      "%s CLIENT: hostname is sent to starter", g_prefix);

        // we should finish the procedure despite of Apache restart
        rc = wait_for_io_or_timeout(g_ap_read_pipe, 1, 1);
        if (APR_STATUS_IS_TIMEUP(rc))
        {
            errptr = ": error on waiting on reading from client pipe";
            goto exit;
        }
        if (rc != APR_SUCCESS)
        {
            errptr = ": timeout exceeded on waiting for reading from client pipe";
            goto exit;
        }

        /* rc == APR_SUCCESS */
        msgsize = sizeof(is_hashed);

        rc = apr_file_read(g_ap_read_pipe, &is_hashed, &msgsize);
        if (rc != APR_SUCCESS)
        {
            errptr = ": error on reading HASHED_FLAG from client pipe";
            goto exit;
        }

        if (is_hashed == 0)
        {

            lscapi_notice(cfg, rc, r->server,
                          "%s CLIENT: starter response received: info is NOT cached for %s", g_prefix, r->hostname);

            rc = wait_for_io_or_timeout(g_ap_write_pipe, 0, 1);
            if (APR_STATUS_IS_TIMEUP(rc))
            {
                errptr = ": timeout exceeded on waiting for writing command into starter pipe";
                goto exit;
            }
            if (rc != APR_SUCCESS)
            {
                errptr = ": error on waiting for writing command into starter pipe";
                goto exit;
            }

            /* rc == APR_SUCCESS */
            rc = apr_file_write_full(g_ap_write_pipe, &command, sizeof command, NULL);
            if (rc != APR_SUCCESS)
            {
                errptr = ": error on writing command into starter pipe";
                goto exit;
            }

            lscapi_notice(cfg, rc, r->server,
                          "%s CLIENT: cached mode spawnCmd is sent to starter for: %s", g_prefix, r->hostname);
        } else
        {
            lscapi_notice(cfg, rc, r->server,
                          "%s CLIENT: starter response received: info is ALREADY cached for %s", g_prefix, r->hostname);
        }
    } // end of else, if (!use_starter_hash_t_structures)

    // we should finish the procedure despite of Apache restart
    rc = wait_for_io_or_timeout(g_ap_read_pipe, 1, 1);
    if (APR_STATUS_IS_TIMEUP(rc))
    {
        errptr = ": timeout exceeded on waiting for reading Notify from client pipe";
        goto exit;
    }
    if (rc != APR_SUCCESS)
    {
        errptr = ": error on waiting for reading Notify from client pipe";
        goto exit;
    }

    /* rc == APR_SUCCESS */
    msgsize = sizeof notify;

    rc = apr_file_read(g_ap_read_pipe, &notify, &msgsize);
    if (rc != APR_SUCCESS)
    {
        errptr = ": apr_file_read_2 on read_pipe failed";
        goto exit;
    }

    lscapi_notice(cfg, rc, r->server,
                  "%s CLIENT notify is received for: %s", g_prefix, r->hostname);
exit:
/* Do we need any special processing of Apache restarting in client? */
/*
    if(starter_should_exit())
    {
    }
*/

    if (rc != APR_SUCCESS)
    {
        notify.pid = -1;
        notify.err = rc;
    }

    /* Release the lock */
    if ((rc = apr_global_mutex_unlock(g_pipelock)) != APR_SUCCESS)
    {
        lscapi_rlog(APLOG_WARNING, rc, r,
                      "%s can't release pipe mutex", g_prefix);
    }

    if (notify.pid < 0)
    {
        lscapi_rlog(APLOG_ERR, notify.err, r,
                      "%s Backend spawn failed%s", g_prefix, errptr);
        sleep(1);
        return -1;
    }

    return notify.pid;
}
