ai2_kit.core.executor module#

pydantic model ai2_kit.core.executor.BaseExecutorConfig[source]#

Bases: BaseModel

Show JSON schema
{
   "title": "BaseExecutorConfig",
   "type": "object",
   "properties": {
      "ssh": {
         "anyOf": [
            {
               "$ref": "#/$defs/SshConfig"
            },
            {
               "type": "null"
            }
         ],
         "default": null
      },
      "queue_system": {
         "$ref": "#/$defs/QueueSystemConfig"
      },
      "work_dir": {
         "title": "Work Dir",
         "type": "string"
      },
      "python_cmd": {
         "default": "python",
         "title": "Python Cmd",
         "type": "string"
      }
   },
   "$defs": {
      "LSF": {
         "additionalProperties": false,
         "properties": {
            "bsub_bin": {
               "default": "bsub",
               "title": "Bsub Bin",
               "type": "string"
            },
            "bjobs_bin": {
               "default": "bjobs",
               "title": "Bjobs Bin",
               "type": "string"
            },
            "polling_interval": {
               "default": 10,
               "title": "Polling Interval",
               "type": "integer"
            }
         },
         "title": "LSF",
         "type": "object"
      },
      "PBS": {
         "additionalProperties": false,
         "properties": {
            "qsub_bin": {
               "default": "qsub",
               "title": "Qsub Bin",
               "type": "string"
            },
            "qstat_bin": {
               "default": "qstat",
               "title": "Qstat Bin",
               "type": "string"
            },
            "qdel_bin": {
               "default": "qdel",
               "title": "Qdel Bin",
               "type": "string"
            }
         },
         "title": "PBS",
         "type": "object"
      },
      "QueueSystemConfig": {
         "additionalProperties": false,
         "properties": {
            "slurm": {
               "anyOf": [
                  {
                     "$ref": "#/$defs/Slurm"
                  },
                  {
                     "type": "null"
                  }
               ],
               "default": null
            },
            "lsf": {
               "anyOf": [
                  {
                     "$ref": "#/$defs/LSF"
                  },
                  {
                     "type": "null"
                  }
               ],
               "default": null
            },
            "pbs": {
               "anyOf": [
                  {
                     "$ref": "#/$defs/PBS"
                  },
                  {
                     "type": "null"
                  }
               ],
               "default": null
            }
         },
         "title": "QueueSystemConfig",
         "type": "object"
      },
      "Slurm": {
         "additionalProperties": false,
         "properties": {
            "sbatch_bin": {
               "default": "sbatch",
               "title": "Sbatch Bin",
               "type": "string"
            },
            "squeue_bin": {
               "default": "squeue",
               "title": "Squeue Bin",
               "type": "string"
            },
            "scancel_bin": {
               "default": "scancel",
               "title": "Scancel Bin",
               "type": "string"
            },
            "polling_interval": {
               "default": 10,
               "title": "Polling Interval",
               "type": "integer"
            }
         },
         "title": "Slurm",
         "type": "object"
      },
      "SshConfig": {
         "additionalProperties": false,
         "properties": {
            "host": {
               "title": "Host",
               "type": "string"
            },
            "gateway": {
               "anyOf": [
                  {
                     "$ref": "#/$defs/SshConfig"
                  },
                  {
                     "type": "null"
                  }
               ],
               "default": null
            }
         },
         "required": [
            "host"
         ],
         "title": "SshConfig",
         "type": "object"
      }
   },
   "additionalProperties": false,
   "required": [
      "queue_system",
      "work_dir"
   ]
}

Config:

  • extra: str = forbid

Fields:
field python_cmd: str = 'python'#
field queue_system: QueueSystemConfig [Required]#
field ssh: SshConfig | None = None#
field work_dir: str [Required]#
class ai2_kit.core.executor.Executor[source]#

Bases: ABC

abstract download(from_path: str, to_dir: str) → str[source]#
abstract dump_text(text: str, path: str)[source]#
abstract glob(pattern: str) → List[str][source]#
init()[source]#
abstract load_text(path: str) → str[source]#
abstract mkdir(path: str)[source]#
name: str#
python_cmd: str#
abstract resolve_artifact(artifact: Artifact) → List[str][source]#
abstract run(script: str, **kwargs) → Result[source]#
abstract run_python_fn(fn: FnType, python_cmd=None, cwd=None) → FnType[source]#
abstract run_python_script(script: str, python_cmd=None)[source]#
setup_workspace(workspace_dir: str, dirs: List[str])[source]#
abstract submit(script: str, **kwargs) → JobFuture[source]#
tmp_dir: str#
abstract upload(from_path: str, to_dir: str) → str[source]#
work_dir: str#
class ai2_kit.core.executor.ExecutorManager(executor_configs: Mapping[str, BaseExecutorConfig])[source]#

Bases: object

get_executor(name: str)[source]#
class ai2_kit.core.executor.HpcExecutor(connector: BaseConnector, queue_system: BaseQueueSystem, work_dir: str, python_cmd: str, name: str)[source]#

Bases: Executor

download(from_path: str, to_dir: str)[source]#
dump_text(text: str, path: str)[source]#
classmethod from_config(config: dict | BaseExecutorConfig, name: str = '')[source]#
glob(pattern: str)[source]#
init()[source]#
property is_local#
load_text(path: str) → str[source]#
mkdir(path: str)[source]#
resolve_artifact(artifact: Artifact) → List[str][source]#
run(script: str, **kwargs)[source]#
run_python_fn(fn: FnType, python_cmd=None, cwd=None) → FnType[source]#
run_python_script(script: str, python_cmd=None, cwd=None)[source]#
submit(script: str, cwd: str, **kwargs)[source]#
upload(from_path: str, to_dir: str)[source]#
upload_python_pkg(pkg: str)[source]#

upload python package to remote server

ai2_kit.core.executor.create_executor(config: BaseExecutorConfig, name: str) → Executor[source]#
ai2_kit.core.executor.fn_to_script(fn: Callable, args, kwargs, delimiter='@')[source]#
ai2_kit.core.executor.pickle_converts(obj, pickle_module='cp', bz2_module='bz2', base64_module='base64')[source]#

convert an object to its pickle string form

On this page
Show Source