{"cells":[{"cell_type":"markdown","metadata":{"id":"view-in-github"},"source":["\"Open"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"SiiXJ7K_fePG"},"source":["

\n"," \n"," \n"," \"sdfstudio\"\n"," \n","

\n","\n","\n","# A Unified Framework for Surface Reconstruction\n","\n","This colab shows how to train and view SDFStudio both on pre-made datasets or from your own videos/images/polycam scan.\n","\n","\\\\\n","\n","Credit to [NeX](https://nex-mpi.github.io/) for Google Colab format."]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":33888,"status":"ok","timestamp":1673975985311,"user":{"displayName":"Refik Soyak","userId":"05459827075173583500"},"user_tz":-60},"id":"RGr33zHaHak0","outputId":"de218bb0-e627-4a60-b419-5eea482e10bd"},"outputs":[],"source":["#@markdown

Install Conda (requires runtime restart)

\n","\n","!pip install -q condacolab\n","import condacolab\n","condacolab.install()"]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"elapsed":1249299,"status":"ok","timestamp":1673977234599,"user":{"displayName":"Refik Soyak","userId":"05459827075173583500"},"user_tz":-60},"id":"9oyLHl8QfYwP","outputId":"cae4ec19-2cb5-48ff-a2df-5b2775e416ad"},"outputs":[],"source":["#@markdown

Install SDFStudio and Dependencies (~15 min)

\n","\n","%cd /content/\n","!pip install --upgrade pip\n","!pip install timm pytorch_lightning torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html\n","\n","# Installing TinyCuda\n","%cd /content/\n","!gdown \"https://drive.google.com/u/1/uc?id=1q8fuc-Mqiev5GTBTRA5UPgCaQDzuqKqj\" \n","!pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch\n","\n","# Installing COLMAP\n","%cd /content/\n","!conda install -c conda-forge colmap\n","\n","# Install SDFStudio\n","%cd /content/\n","!git clone https://github.com/autonomousvision/sdfstudio.git\n","%cd sdfstudio\n","!pip install --upgrade pip setuptools\n","!pip install -e .\n","# install tab completion\n","!ns-install-cli\n","\n","# Install omnidata and pretrained models\n","%cd /content/\n","!git clone https://github.com/EPFL-VILAB/omnidata.git\n","%cd /content/omnidata/omnidata_tools/torch\n","!mkdir -p pretrained_models \n","!gdown '1Jrh-bRnJEjyMCS7f-WsaFlccfPjJPPHI&confirm=t' -O ./pretrained_models/ # omnidata depth (v2)\n","!gdown '1wNxVO4vVbDEMEpnAi_jwQObf2MFodcBR&confirm=t' -O ./pretrained_models/ # omnidata normals (v2)"]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","colab":{"base_uri":"https://localhost:8080/","height":924},"executionInfo":{"elapsed":830199,"status":"ok","timestamp":1673991336787,"user":{"displayName":"Refik Soyak","userId":"05459827075173583500"},"user_tz":-60},"id":"msVLprI4gRA4","outputId":"b8ed98d7-f96d-4a4b-c45c-dd170d0e38ed"},"outputs":[],"source":["#@markdown

Downloading and Processing Data

\n","#@markdown

Pick the demo data or upload your own images/video and process

\n","import os\n","import glob\n","from google.colab import files\n","from IPython.core.display import display, HTML\n","\n","scene = '\\uD83D\\uDCE4 upload your images' #@param ['💀 dtu-scan65', '🏠 replica-room0', '📤 upload your images' , '🎥 upload your own video', '🔺 upload Polycam data']\n","#@markdown

Select scene type if you upload your own data

\n","scene_type = \"object\" #@param [\"indoor\", \"object\", \"unbound\"] {allow-input: true}\n","\n","scene = ' '.join(scene.split(' ')[1:])\n","sdf_data_dir = None\n","\n","if scene in ['dtu-scan65', 'replica-room0']:\n"," %cd /content/sdfstudio\n"," !python scripts/downloads/download_data.py sdfstudio\n"," sdf_data_dir = f\"/content/sdfstudio/data/sdfstudio-demo-data/{scene}\"\n","else:\n"," data_type = None\n"," pre_data_dir = None\n"," if scene == \"upload Polycam data\":\n"," data_type = \"polycam\"\n"," %cd /content/\n"," !mkdir -p /content/data/nerfstudio/custom_data\n"," %cd /content/data/nerfstudio/custom_data/\n"," uploaded = files.upload()\n"," dir = os.getcwd()\n"," if len(uploaded.keys()) > 1:\n"," print(\"ERROR, upload a single .zip file when processing Polycam data\")\n"," pre_data_dir = [os.path.join(dir, f) for f in uploaded.keys()][0]\n"," elif scene in ['upload your images', 'upload your own video']:\n"," data_type = \"colmap\"\n"," display(HTML('

Select your custom data

'))\n"," display(HTML('

You can select multiple images by pressing ctrl, cmd or shift and click.

'))\n"," display(HTML('

Note: This may take time, especially on hires inputs, so we recommend to download dataset after creation.

'))\n"," !mkdir -p /content/data/nerfstudio/custom_data\n"," if scene == 'upload your images':\n"," !mkdir -p /content/data/nerfstudio/custom_data/raw_images\n"," %cd /content/data/nerfstudio/custom_data/raw_images\n"," uploaded = files.upload()\n"," dir = os.getcwd()\n"," else:\n"," %cd /content/data/nerfstudio/custom_data/\n"," uploaded = files.upload()\n"," dir = os.getcwd()\n"," preupload_datasets = [os.path.join(dir, f) for f in uploaded.keys()]\n"," del uploaded\n"," %cd /content/\n","\n"," pre_data_dir = \"/content/data/nerfstudio/custom_data/\"\n"," if scene == 'upload your images':\n"," !ns-process-data images --data /content/data/nerfstudio/custom_data/raw_images --output-dir $pre_data_dir\n"," else:\n"," video_path = preupload_datasets[0]\n"," !ns-process-data video --data $video_path --output-dir $pre_data_dir\n","\n"," scene = \"custom_data\"\n"," sdf_data_dir = \"/content/sdfstudio/data/custom_data\"\n"," %cd /content/sdfstudio/\n"," !python scripts/datasets/process_nerfstudio_to_sdfstudio.py \\\n"," --data-type $data_type --scene-type $scene_type \\\n"," --data $pre_data_dir \\\n"," --output-dir $sdf_data_dir\n","\n","print(\"Data processing succeeded!\")"]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","colab":{"base_uri":"https://localhost:8080/","height":929},"executionInfo":{"elapsed":9326,"status":"ok","timestamp":1673991368302,"user":{"displayName":"Refik Soyak","userId":"05459827075173583500"},"user_tz":-60},"id":"VoKDxqEcjmfC","outputId":"d524239d-f5da-46f5-b744-d3350873d70e"},"outputs":[],"source":["#@markdown

Set up and Start Viewer

\n","\n","%cd /content\n","\n","# Install localtunnel\n","# We are using localtunnel https://github.com/localtunnel/localtunnel but ngrok could also be used\n","!npm install -g localtunnel\n","\n","# Tunnel port 7007, the default for\n","!rm url.txt 2> /dev/null\n","get_ipython().system_raw('lt --port 7007 >> url.txt 2>&1 &')\n","\n","import time\n","time.sleep(3) # the previous command needs time to write to url.txt\n","\n","\n","with open('url.txt') as f:\n"," lines = f.readlines()\n","websocket_url = lines[0].split(\": \")[1].strip().replace(\"https\", \"wss\")\n","# from nerfstudio.utils.io import load_from_json\n","# from pathlib import Path\n","# json_filename = \"nerfstudio/nerfstudio/viewer/app/package.json\"\n","# version = load_from_json(Path(json_filename))[\"version\"]\n","url = f\"https://viewer.nerf.studio/?websocket_url={websocket_url}\"\n","print(url)\n","print(\"You may need to click Refresh Page after you start training!\")\n","from IPython import display\n","display.IFrame(src=url, height=800, width=\"100%\")"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":73996,"status":"ok","timestamp":1673983652959,"user":{"displayName":"Refik Soyak","userId":"05459827075173583500"},"user_tz":-60},"id":"ag3al57qLIzi","outputId":"b43bed08-9b6c-4b77-abaf-46b0c9ea347f"},"outputs":[],"source":["!conda remove --force qt-main"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"m_N8_cLfjoXD","outputId":"3bbedfaa-6773-4c46-fbb7-75325261f9f4"},"outputs":[],"source":["#@markdown

Start Training

\n","%cd /content/sdfstudio/\n","!ns-train neus-facto --pipeline.model.sdf-field.inside-outside False \\\n","--vis viewer --experiment-name neus-facto-$scene sdfstudio-data \\\n","--data $sdf_data_dir --auto-orient True"]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","id":"aYjsiMp4v7mA"},"outputs":[],"source":["#@markdown

Extract Mesh

\n","config_file_path = \"/content/sdfstudio/outputs/neus-facto-dtu-scan65/neus-facto/2023-01-17_200838/config.yml\" #@param {type:\"string\"}\n","output_path = \"./meshes/neus-facto-dtu65.ply\" #@param {type:\"string\"}\n","\n","%cd /content/sdfstudio\n","!ns-extract-mesh --load-config $config_file_path --output-path $output_path"]},{"cell_type":"code","execution_count":null,"metadata":{"cellView":"form","id":"szlByjPJw80T"},"outputs":[],"source":["#@markdown

Texture Mesh with Nerf

\n","target_num_faces = 10000 #@param {type:\"integer\"}\n","config_file_path = \"/content/sdfstudio/outputs/neus-facto-dtu-scan65/neus-facto/2023-01-17_200838/config.yml\" #@param {type:\"string\"}\n","mesh_file_path = \"meshes/neus-facto-dtu65.ply\" #@param {type:\"string\"}\n","output_dir = \"./textures\" #@param {type:\"string\"}\n","\n","%cd /content/sdfstudio/\n","!python scripts/texture.py --load-config $config_file_path --input-mesh-filename $mesh_file_path --output-dir $output_dir --target_num_faces $target_num_faces"]}],"metadata":{"accelerator":"GPU","colab":{"provenance":[]},"gpuClass":"standard","kernelspec":{"display_name":"Python 3.8.13 ('nerfstudio')","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.8.13"},"vscode":{"interpreter":{"hash":"c59f626636933ef1dc834fb3684b382f705301c5306cf8436d2da634c2289783"}}},"nbformat":4,"nbformat_minor":0}