featureDB_make_labels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cc3d\n",
    "import netCDF4\n",
    "import glob\n",
    "import numpy\n",
    "import sys\n",
    "import datetime\n",
    "import pickle\n",
    "import time\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/tablespace/xcal/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_imerg(file_path, variable_name):\n",
    "    netcdf = netCDF4.Dataset(file_path, 'r', format='NETCDF4')\n",
    "    data = netcdf.groups['Grid'][variable_name][:][0].T[::-1]\n",
    "    file_header = netcdf.FileHeader.split(';\\n')\n",
    "    file_header.remove('')\n",
    "    header = {r.split('=')[0]: r.split('=')[1] for r in file_header}\n",
    "    return data, header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\r"
     ]
    }
   ],
   "source": [
    "file_paths = sorted(glob.glob(data_dir + 'imerg2022/3B-HHR*'))\n",
    "variable_name = 'precipitationCal'\n",
    "\n",
    "stack = []\n",
    "headers = []\n",
    "timestamps = []\n",
    "for file_path in file_paths:\n",
    "    print('{}'.format(len(file_paths)-len(stack)), end='\\r')\n",
    "    sys.stdout.flush()\n",
    "    data, header = load_imerg(file_path, variable_name)\n",
    "    stack.append(data)\n",
    "    headers.append(header)\n",
    "    timestamps.append(header['StartGranuleDateTime'])\n",
    "    \n",
    "data = numpy.array(stack)\n",
    "\n",
    "timestamps_dt = [datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.000Z') for date in timestamps]\n",
    "timestamps_np = numpy.array(timestamps_dt, dtype='datetime64[m]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data = numpy.flip(data, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('{}/pickles/data.pickle'.format(data_dir), 'wb') as f:\n",
    "    pickle.dump(data, f)\n",
    "    \n",
    "with open('{}/pickles/timestamps.pickle'.format(data_dir), 'wb') as f:\n",
    "    pickle.dump(timestamps_dt, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('{}/pickles/data.pickle'.format(data_dir), 'rb') as f:\n",
    "    imerg = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Thresholding and CCLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = imerg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresh = 0.5\n",
    "data[data<thresh] = 0\n",
    "data[data>=thresh] = 1\n",
    "\n",
    "connectivity = 6 \n",
    "min_voxels = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64.06008338928223\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "labels, N = cc3d.connected_components(data, delta=0,connectivity=connectivity, return_N=True)\n",
    "print(time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2376.461284637451\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "cc3d.dust(labels, threshold=min_voxels, connectivity=connectivity, in_place=True)\n",
    "print(time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8883,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_names = numpy.unique(labels[labels>0])\n",
    "label_names.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3922.8200602531433\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "largest_20, N = cc3d.largest_k(labels, k=20, connectivity=connectivity, delta=0, return_N=True)\n",
    "largest_100, N = cc3d.largest_k(labels, k=100, connectivity=connectivity, delta=0, return_N=True)\n",
    "print(time.time() - start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('{}/pickles/largest_20.pickle'.format(data_dir), 'wb') as f:\n",
    "    pickle.dump(largest_20, f)\n",
    "    \n",
    "with open('{}/pickles/largest_100.pickle'.format(data_dir), 'wb') as f:\n",
    "    pickle.dump(largest_100, f)\n",
    "\n",
    "with open('{}/pickles/labels.pickle'.format(data_dir), 'wb') as f:\n",
    "    pickle.dump(labels, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ccl2",
   "language": "python",
   "name": "ccl2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}