pulsar.managers package

Subpackages

• pulsar.managers.base package
  • Submodules
  • pulsar.managers.base.base_drmaa module
    • BaseDrmaaManager
      • BaseDrmaaManager.shutdown()
  • pulsar.managers.base.directory module
    • DirectoryBaseManager
      • DirectoryBaseManager.job_stderr_contents()
      • DirectoryBaseManager.job_stdout_contents()
      • DirectoryBaseManager.read_command_line()
      • DirectoryBaseManager.return_code()
      • DirectoryBaseManager.stderr_contents()
      • DirectoryBaseManager.stdout_contents()
  • pulsar.managers.base.external module
    • ExternalBaseManager
      • ExternalBaseManager.clean()
      • ExternalBaseManager.get_status()
      • ExternalBaseManager.kill()
  • Module contents
    • BaseManager
      • BaseManager.clean()
      • BaseManager.job_directory()
      • BaseManager.setup_job()
      • BaseManager.system_properties()
    • DirectoryMaker
      • DirectoryMaker.make()
    • JobDirectory
      • JobDirectory.calculate_path()
      • JobDirectory.contains_file()
      • JobDirectory.delete()
      • JobDirectory.exists()
      • JobDirectory.has_metadata()
      • JobDirectory.job_directory_contents()
      • JobDirectory.load_metadata()
      • JobDirectory.lock()
      • JobDirectory.make_directory()
      • JobDirectory.metadata_directory_contents()
      • JobDirectory.open_file()
      • JobDirectory.outputs_directory_contents()
      • JobDirectory.read_file()
      • JobDirectory.remove_file()
      • JobDirectory.remove_metadata()
      • JobDirectory.setup()
      • JobDirectory.store_metadata()
      • JobDirectory.working_directory_contents()
      • JobDirectory.working_directory_file_contents()
      • JobDirectory.write_file()
    • get_id_assigner()
• pulsar.managers.staging package
  • Submodules
  • pulsar.managers.staging.post module
    • postprocess()
    • realized_dynamic_file_sources()
  • pulsar.managers.staging.pre module
    • preprocess()
  • Module contents
    • postprocess()
    • preprocess()
    • realized_dynamic_file_sources()
• pulsar.managers.util package
  • Subpackages
    • pulsar.managers.util.cli package
      • Subpackages
        • pulsar.managers.util.cli.job package
          • Submodules
          • pulsar.managers.util.cli.job.lsf module
            • LSF
              • LSF.delete()
              • LSF.get_failure_reason()
              • LSF.get_single_status()
              • LSF.get_status()
              • LSF.job_script_kwargs()
              • LSF.parse_failure_reason()
              • LSF.parse_single_status()
              • LSF.parse_status()
              • LSF.submit()
          • pulsar.managers.util.cli.job.pbs module
            • OpenPBS
              • OpenPBS.ERROR_MESSAGE_UNRECOGNIZED_ARG
              • OpenPBS.get_single_status()
              • OpenPBS.get_status()
              • OpenPBS.parse_status()
          • pulsar.managers.util.cli.job.slurm module
            • Slurm
              • Slurm.delete()
              • Slurm.get_single_status()
              • Slurm.get_status()
              • Slurm.job_script_kwargs()
              • Slurm.parse_single_status()
              • Slurm.parse_status()
              • Slurm.submit()
          • pulsar.managers.util.cli.job.slurm_torque module
            • SlurmTorque
              • SlurmTorque.get_status()
              • SlurmTorque.parse_status()
          • pulsar.managers.util.cli.job.torque module
            • Torque
              • Torque.ERROR_MESSAGE_UNRECOGNIZED_ARG
              • Torque.delete()
              • Torque.get_single_status()
              • Torque.get_status()
              • Torque.job_script_kwargs()
              • Torque.parse_single_status()
              • Torque.parse_status()
              • Torque.submit()
          • Module contents
            • BaseJobExec
              • BaseJobExec.delete()
              • BaseJobExec.get_failure_reason()
              • BaseJobExec.get_single_status()
              • BaseJobExec.get_status()
              • BaseJobExec.job_script_kwargs()
              • BaseJobExec.parse_failure_reason()
              • BaseJobExec.parse_single_status()
              • BaseJobExec.parse_status()
              • BaseJobExec.submit()
            • job_states
              • job_states.ERROR
              • job_states.OK
              • job_states.QUEUED
              • job_states.RUNNING
        • pulsar.managers.util.cli.shell package
          • Submodules
          • pulsar.managers.util.cli.shell.local module
            • LocalShell
              • LocalShell.execute()
          • pulsar.managers.util.cli.shell.rsh module
            • GlobusSecureShell
            • ParamikoShell
              • ParamikoShell.connect()
              • ParamikoShell.execute()
            • RemoteShell
              • RemoteShell.execute()
            • SecureShell
          • Module contents
            • BaseShellExec
              • BaseShellExec.execute()
      • Submodules
      • pulsar.managers.util.cli.factory module
        • build_cli_interface()
        • get_shell()
      • Module contents
        • CliInterface
          • CliInterface.get_job_interface()
          • CliInterface.get_plugins()
          • CliInterface.get_shell_plugin()
        • split_params()
    • pulsar.managers.util.condor package
      • Module contents
        • build_submit_description()
        • condor_stop()
        • condor_submit()
        • submission_params()
        • summarize_condor_log()
    • pulsar.managers.util.drmaa package
      • Module contents
        • DrmaaSessionFactory
          • DrmaaSessionFactory.get()
    • pulsar.managers.util.job_script package
      • Module contents
        • job_script()
        • write_script()
  • Submodules
  • pulsar.managers.util.aws_batch module
    • ensure_client_available()
    • get_client()
  • pulsar.managers.util.env module
    • env_to_statement()
  • pulsar.managers.util.external module
    • parse_external_id()
  • pulsar.managers.util.kill module
    • kill_pid()
  • pulsar.managers.util.process_groups module
    • check_pg()
    • kill_pg()
  • pulsar.managers.util.pykube_util module
    • ensure_pykube()
    • find_job_object_by_name()
    • find_pod_object_by_name()
    • galaxy_instance_id()
    • job_object_dict()
    • produce_unique_k8s_job_name()
    • pull_policy()
    • pykube_client_from_dict()
    • stop_job()
  • pulsar.managers.util.retry module
    • RetryActionExecutor
      • RetryActionExecutor.execute()
  • pulsar.managers.util.sudo module
    • sudo_popen()
  • pulsar.managers.util.tes module
    • TesClient
      • TesClient.cancel_task()
      • TesClient.create_task()
      • TesClient.get_task()
      • TesClient.list_tasks()
      • TesClient.service_info()
    • TesExecutor
      • TesExecutor.command
      • TesExecutor.env
      • TesExecutor.image
      • TesExecutor.stderr
      • TesExecutor.stdin
      • TesExecutor.stdout
      • TesExecutor.workdir
    • TesResources
      • TesResources.backend_parameters
      • TesResources.backend_parameters_strict
      • TesResources.cpu_cores
      • TesResources.disk_gb
      • TesResources.preemptible
      • TesResources.ram_gb
      • TesResources.zones
    • TesState
      • TesState.CANCELED
      • TesState.COMPLETE
      • TesState.EXECUTOR_ERROR
      • TesState.INITIALIZING
      • TesState.PAUSED
      • TesState.QUEUED
      • TesState.RUNNING
      • TesState.SYSTEM_ERROR
      • TesState.UNKNOWN
    • TesTask
      • TesTask.creation_time
      • TesTask.description
      • TesTask.executors
      • TesTask.id
      • TesTask.inputs
      • TesTask.logs
      • TesTask.name
      • TesTask.outputs
      • TesTask.resources
      • TesTask.state
      • TesTask.tags
      • TesTask.volumes
    • ensure_tes_client()
  • Module contents
    • kill_pid()

Submodules

pulsar.managers.queued module

class pulsar.managers.queued.QueueManager(name, app, **kwds)

Bases: Manager

A job manager that queues up jobs directly (i.e. does not use an external queuing software such PBS, SGE, etc…).

launch(job_id, command_line, submit_params=None, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_python'

run_next()

Run the next item in the queue (a job waiting to run).

shutdown(timeout=None)

pulsar.managers.queued_cli module

Pulsar job manager that uses a CLI interface to a job queue (e.g. Torque’s qsub, qstat, etc…).

class pulsar.managers.queued_cli.CliQueueManager(name, app, **kwds)

Bases: ExternalBaseManager

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_cli'

pulsar.managers.queued_condor module

class pulsar.managers.queued_condor.CondorQueueManager(name, app, **kwds)

Bases: ExternalBaseManager

Job manager backend that plugs into Condor.

get_status(job_id)

Return status of job as string, currently supported statuses include ‘cancelled’, ‘running’, ‘queued’, and ‘complete’.

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_condor'

pulsar.managers.queued_drmaa module

class pulsar.managers.queued_drmaa.DrmaaQueueManager(name, app, **kwds)

Bases: BaseDrmaaManager

DRMAA backed queue manager.

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_drmaa'

pulsar.managers.queued_drmaa_xsede module

class pulsar.managers.queued_drmaa_xsede.XsedeDrmaaQueueManager(name, app, **kwds)

Bases: DrmaaQueueManager

DRMAA backed queue manager for XSEDE (to run the XSEDE job/user reporting script after submission).

TODO: A generalized callback framework for executing things at various points in the job lifecycle.

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_drmaa_xsede'

pulsar.managers.queued_drmaa_xsede.check_output(args)

Pipe-safe (and 2.6 compatible) version of subprocess.check_output

pulsar.managers.queued_external_drmaa module

class pulsar.managers.queued_external_drmaa.ExternalDrmaaQueueManager(name, app, **kwds)

Bases: BaseDrmaaManager

DRMAA backed queue manager.

get_status(job_id)

Return status of job as string, currently supported statuses include ‘cancelled’, ‘running’, ‘queued’, and ‘complete’.

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'queued_external_drmaa'

pulsar.managers.queued_pbs module

class pulsar.managers.queued_pbs.PbsQueueManager(name, app, **kwds)

Bases: BaseManager

Placeholder for PBS-python backed queue manager. Not yet implemented, for many situations this would be used the DRMAA or CLI+Torque managers may be better choices or at least stop gaps.

manager_type = 'queued_pbs'

pulsar.managers.stateful module

class pulsar.managers.stateful.StatefulManagerProxy(manager, **manager_options)

Bases: ManagerProxy

get_status(job_id)

Compute status used proxied manager and handle state transitions and track additional state information needed.

handle_failure_before_launch(job_id)

property name

preprocess_and_launch(job_id, launch_config)

recover_active_jobs()

set_state_change_callback(state_change_callback)

setup_job(*args, **kwargs)

shutdown(timeout=None)

Optional.

touch_outputs(job_id, touch_outputs)

trigger_state_change_callback(job_id)

pulsar.managers.status module

pulsar.managers.status.is_job_done(status)

Does the supplied status correspond to a finished job (done processing).

pulsar.managers.unqueued module

class pulsar.managers.unqueued.Manager(name, app, **kwds)

Bases: BaseUnqueuedManager

A simple job manager that just directly runs jobs as given (no queueing). Preserved for compatibilty with older versions of Pulsar client code where Galaxy is used to maintain queue (like Galaxy’s local job runner).

get_status(job_id)

Return status of job as string, currently supported statuses include ‘cancelled’, ‘running’, ‘queued’, and ‘complete’.

kill(job_id)

End or cancel execution of the specified job.

launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

manager_type = 'unqueued'

Module contents

class pulsar.managers.ManagerInterface

Bases: object

Defines the interface to various job managers.

abstract clean(job_id)

Delete job directory and clean up resources associated with job with id job_id.

abstract get_status(job_id)

Return status of job as string, currently supported statuses include ‘cancelled’, ‘running’, ‘queued’, and ‘complete’.

abstract job_directory(job_id)

Return a JobDirectory abstraction describing the state of the job working directory.

abstract job_stderr_contents(job_id)

After completion, return contents of stderr of the job as produced by the job runner.

abstract job_stdout_contents(job_id)

After completion, return contents of stdout of the job as produced by the job runner.

abstract kill(job_id)

End or cancel execution of the specified job.

abstract launch(job_id, command_line, submit_params={}, dependencies_description=None, env=[], setup_params=None)

Called to indicate that the client is ready for this job with specified job id and command line to be executed (i.e. run or queue this job depending on implementation).

abstract return_code(job_id)

Return integer indicating return code of specified execution or PULSAR_UNKNOWN_RETURN_CODE.

abstract setup_job(input_job_id, tool_id, tool_version)

Setup a job directory for specified input (galaxy) job id, tool id, and tool version.

abstract stderr_contents(job_id)

After completion, return contents of stderr of the tool script.

abstract stdout_contents(job_id)

After completion, return contents of stdout of the tool script.

class pulsar.managers.ManagerProxy(manager)

Bases: object

Subclass to build override proxy a manager and override specific functionality.

clean(*args, **kwargs)

get_status(*args, **kwargs)

job_directory(*args, **kwargs)

job_stderr_contents(*args, **kwargs)

job_stdout_contents(*args, **kwargs)

kill(*args, **kwargs)

launch(*args, **kwargs)

property object_store

return_code(*args, **kwargs)

setup_job(*args, **kwargs)

shutdown(timeout=None)

Optional.

stderr_contents(*args, **kwargs)

stdout_contents(*args, **kwargs)

system_properties()