{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# How to use mml interactively\n",
    "\n",
    "`MML` provides the `mml.interactive` module with the `mml.interactive.planning` and `mml.interactive.loading` submodules to support direct usage within jupyter notebooks. This notebook itself demonstrates the usage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:23:56.175656Z",
     "start_time": "2023-05-09T15:23:56.084137Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import mml.interactive\n",
    "\n",
    "# your mml.env location might differ\n",
    "mml.interactive.init(env_path=Path(mml.__file__).parent / \"mml.env\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:27:02.765778Z",
     "start_time": "2023-05-09T15:27:02.589385Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# this example shows how to retrieve all installed tasks\n",
    "with mml.interactive.default_file_manager() as fm:\n",
    "    print(sorted(list(fm.task_index.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:29:05.620904Z",
     "start_time": "2023-05-09T15:29:05.487248Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# this example loads task information\n",
    "tasks = [\"caltech256_object_classification\", \"cifar10_object_classification\", \"svhn\"]\n",
    "mml.interactive.get_task_infos(tasks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## experiment planning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:42:40.699820Z",
     "start_time": "2023-05-09T15:42:40.695104Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# this example creates some MML calls to run experiments\n",
    "base_reqs = mml.interactive.DefaultRequirements()\n",
    "all_cmds = list()\n",
    "# create task data\n",
    "all_cmds.append(\n",
    "    mml.interactive.MMLJobDescription(\n",
    "        prefix_req=base_reqs, mode=\"create\", config_options={\"task_list\": tasks, \"proj\": \"my_exp\"}\n",
    "    )\n",
    ")\n",
    "# preprocess tasks\n",
    "all_cmds.append(\n",
    "    mml.interactive.MMLJobDescription(\n",
    "        prefix_req=base_reqs, mode=\"pp\", config_options={\"task_list\": tasks, \"proj\": \"my_exp\"}\n",
    "    )\n",
    ")\n",
    "# create tagged task variants\n",
    "all_cmds.append(\n",
    "    mml.interactive.MMLJobDescription(\n",
    "        prefix_req=base_reqs,\n",
    "        mode=\"info\",\n",
    "        config_options={\"task_list\": tasks, \"proj\": \"my_exp\", \"tagging.all\": \"+subset?0_05+confuse?0_1\"},\n",
    "    )\n",
    ")\n",
    "# you may also loop over\n",
    "for t in tasks:\n",
    "    all_cmds.append(\n",
    "        mml.interactive.MMLJobDescription(\n",
    "            prefix_req=base_reqs,\n",
    "            mode=\"train\",\n",
    "            config_options={\"pivot.name\": f\"{t}+subset?0_05+confuse?0_1\", \"proj\": \"my_exp\", \"sampling.batch_size\": 500},\n",
    "        )\n",
    "    )\n",
    "# either render jobs directly\n",
    "for cmd in all_cmds:\n",
    "    print(cmd.render())\n",
    "# or put them into a file\n",
    "mml.interactive.write_out_commands(cmd_list=all_cmds, name=\"my_exp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": "## start planned jobs from within a notebook"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runner = mml.interactive.EmbeddedJobRunner()\n",
    "job = mml.interactive.MMLJobDescription(\n",
    "    prefix_req=base_reqs,\n",
    "    mode=\"info\",\n",
    "    config_options={\"task_list\": tasks, \"proj\": \"my_exp\", \"tagging.all\": \"+subset?0_05+confuse?0_1\"},\n",
    ")\n",
    "runner.run(job=job)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## experiment evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:47:34.352483Z",
     "start_time": "2023-05-09T15:47:34.232398Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load experiment models\n",
    "models = mml.interactive.load_project_models(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-09T15:49:56.550298Z",
     "start_time": "2023-05-09T15:49:56.547572Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# inspect and evaluate\n",
    "print(models[\"lapgyn4_instrument_count_miccai\"][0].training_time)\n",
    "print(models[\"lapgyn4_instrument_count_miccai\"][0].performance)\n",
    "print(models[\"lapgyn4_instrument_count_miccai\"][0].metrics[-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}