tpu

.gitignore

@@ -107,6 +107,7 @@ venv.bak/
 config.json
 
 data/
+upload/
 tmp/
 bpe.ipynb
 sentencepiece/
@@ -120,5 +121,4 @@ gpt2/
 Untitled.ipynb
 tpu/.gcp_credentials.json
 .terraform
-tpu/terraform.tfstate
-tpu/terraform.tfstate.backup
+terraform.tfstate

README.md

@@ -58,23 +58,23 @@ export TRAIN_FILE=./data/full
 
 export CUDA_VISIBLE_DEVICES=1
 export MODEL_SIZE=gpt2
-export OUTPUT=output_s
+export OUTPUT=output/s
 export BS=8
 export LR=5e-5
 
 # GPT-2 355M, final perplexity 18.99?
 
 export CUDA_VISIBLE_DEVICES=2
 export MODEL_SIZE=gpt2-medium
-export OUTPUT=output_m
+export OUTPUT=output/m
 export BS=3
 export LR=3e-5
 
 # GPT-2 774M, final perplexity 21.09?
 
 export CUDA_VISIBLE_DEVICES=3
 export MODEL_SIZE=gpt2-large
-export OUTPUT=output_l
+export OUTPUT=output/l
 export BS=1
 export LR=1e-5
 

corpus/corpus.ipynb

@@ -843,6 +843,56 @@
     "    pass"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Prepare cached dataset for upload (for GCloud)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = get_files('upload', '.txt', True); len(files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = [item for item in files if '/full' in str(item) and '/cached' not in str(item)]; len(files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for item in files:\n",
+    "    with open(item, 'w'):\n",
+    "        pass"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

environment.yml

@@ -20,6 +20,8 @@ dependencies:
   - uvicorn
   - terraform
   - google-cloud-sdk
+  - tar 
+  - zstd
   - pip:
     - transformers
     - sentencepiece

tpu/main.tf → tpu/00_prepare/main.tf

@@ -8,34 +8,47 @@ resource "google_compute_address" "ip_address" {
   name = "my-address"
 }
 
+resource "google_compute_disk" "data-disk" {
+  name  = "data-disk"
+  type  = "pd-ssd"
+  size = 200
+  zone  = "us-west1-a"
+}
+
 // A single Google Cloud Engine instance
 resource "google_compute_instance" "default" {
- name         = "train-instance"
- machine_type = "n1-standard-1"
- zone         = "us-west1-a"
-
- boot_disk {
-   initialize_params {
-     image = "ubuntu-1804-lts"
-     size = 20
-   }
- }
-
- metadata_startup_script = "sudo apt update; sudo apt upgrade -y"
-
- tags = ["train"]
-
- metadata = {
+  name         = "train-instance"
+  machine_type = "n1-standard-1"
+  zone         = "us-west1-a"
+
+  boot_disk {
+    device_name = "basic-disk"
+    initialize_params {
+      image = "ubuntu-1804-lts"
+      type = "pd-ssd"
+      size = 200
+    }
+  }
+
+  attached_disk {
+    source = "${google_compute_disk.data-disk.name}"
+  }
+
+  metadata_startup_script = "sudo apt update; sudo apt upgrade -y"
+
+  tags = ["train"]
+
+  metadata = {
     ssh-keys = "ubuntu:${file("~/.ssh/id_rsa.pub")}"
   }
 
- network_interface {
-   network = "open-network"
+  network_interface {
+    network = "open-network"
 
-   access_config {
-       nat_ip = "${google_compute_address.ip_address.address}"
-   }
- }
+    access_config {
+        nat_ip = "${google_compute_address.ip_address.address}"
+    }
+  }
 }
 
 resource "google_compute_firewall" "default" {

tpu/README.md

@@ -30,6 +30,7 @@ gcloud iam service-accounts get-iam-policy \
 ### Create instance with Terraform
 
 ```
+cd 00_prepare/
 terraform init
 terraform plan
 terraform apply
@@ -40,6 +41,23 @@ terraform apply
 ```
 IP=35.185.201.94 # your node IP
 scp train_setup.sh ubuntu@$IP:
-ssh ubuntu@$IP bash ./train_setup.sh
+# dataset packed with 'tar -caf data.zst.tar data/'
+rsync -vP data.zst.tar ubuntu@$IP:  
+
+# go there 
+ssh ubuntu@$IP 
+
+sudo mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard /dev/sdb
+bash ./train_setup.sh
+sudo -s
+crontab -l | { cat; echo "@reboot mount /dev/sdb /home/ubuntu/ru_transformers/output"; } | crontab -
+exit
+
 ```
 
+### Create an image for preemptive instance
+
+```
+gcloud compute images create train-image --source-disk train-instance --source-disk-zone us-west1-a --force
+#gcloud compute images delete train-image 
+```

tpu/train_setup.sh

@@ -11,6 +11,9 @@ source .bashrc
 git clone https://github.com/mgrankin/ru_transformers
 cd ru_transformers
 conda env create -f environment.yml
+mkdir ru_transformers/output
 
 cd
-rm train_setup.sh
+tar -xaf data.zst.tar
+rm data.zst.tar
+mv data ru_transformers/