method_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "from common.constants import file_paths\n",
    "from common.constants import file_suffixes\n",
    "from common.constants import regex_patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONCEPTS_SIZE = 800\n",
    "SAMPLE_SIZE = 100\n",
    "\n",
    "HAND_LABEL_ECON_SAMPLES = False\n",
    "HAND_LABEL_AUTOPHRASE_SAMPLES = False\n",
    "HAND_LABEL_PRDR_SAMPLES = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 1900 good phrases.\n"
     ]
    }
   ],
   "source": [
    "good_phrases = set(pickle.load(open('data/positive_samples.pkl', 'rb')))\n",
    "print(f'Loaded {len(good_phrases)} good phrases.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_path(file_suffix):\n",
    "    extensionless_input_file_name = file_paths.INPUT_FILE.split('.')[0]\n",
    "    return f'{file_paths.DATA_DIR}/{extensionless_input_file_name}{file_suffix}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_embedding_path = get_file_path(file_suffixes.ECON_EMBEDDING)\n",
    "autophrase_results_path = get_file_path(file_suffixes.AUTOPHRASE_RESULTS)\n",
    "prdr_results_path = get_file_path('_prdr_results.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_raw_concept(tagged_econ_concept):\n",
    "    return tagged_econ_concept.replace('_', ' ')[3:-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_ECON_SAMPLES:\n",
    "    model = Word2Vec.load(econ_embedding_path)\n",
    "    econ_concepts = [get_raw_concept(w) for w in model.wv.index2word if regex_patterns.CONCEPT_TAGGED_PHRASE.match(w)]\n",
    "    print(f'Loaded {len(econ_concepts)} ECON concepts.')\n",
    "    econ_samples = random.sample(econ_concepts, SAMPLE_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_AUTOPHRASE_SAMPLES:\n",
    "    with open(autophrase_results_path, 'r') as f:\n",
    "        autophrase_concepts = f.read().split('\\n')[:-1]\n",
    "        f.close()\n",
    "    print(f'Loaded {len(autophrase_concepts)} AutoPhrase concepts.')\n",
    "    autophrase_samples = random.sample(autophrase_concepts, SAMPLE_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_PRDR_SAMPLES:\n",
    "    prdr_concepts = [line.split('\\t')[0] for line in open(prdr_results_path, 'r').read().split('\\n')[:-1]]\n",
    "    print(f'Loaded {len(prdr_concepts)} concepts extracted via PRDR.')\n",
    "    prdr_samples = random.sample(prdr_concepts, SAMPLE_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_ECON_SAMPLES:\n",
    "    labeled_econ_samples = []\n",
    "    num_hl_positive = 0\n",
    "    num_gp_positive = 0\n",
    "    for sample in econ_samples:\n",
    "        label = input(f'{sample}: ')\n",
    "        labeled_econ_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "        if sample in good_phrases:\n",
    "            num_gp_positive += 1\n",
    "    pickle.dump(labeled_econ_samples, open('data/labeled_econ_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_AUTOPHRASE_SAMPLES:\n",
    "    labeled_autophrase_samples = []\n",
    "    num_hl_positive = 0\n",
    "    num_gp_positive = 0\n",
    "    for sample in autophrase_samples:\n",
    "        label = input(f'{sample}: ')\n",
    "        labeled_autophrase_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "        if sample in good_phrases:\n",
    "            num_gp_positive += 1\n",
    "    pickle.dump(labeled_autophrase_samples, open('data/labeled_autophrase_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HAND_LABEL_PRDR_SAMPLES:\n",
    "    labeled_prdr_samples = []\n",
    "    num_hl_positive = 0\n",
    "    num_gp_positive = 0\n",
    "    for sample in prdr_samples:\n",
    "        label = input(f'{sample}: ')\n",
    "        labeled_prdr_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "        if sample in good_phrases:\n",
    "            num_gp_positive += 1\n",
    "    pickle.dump(labeled_prdr_samples, open('data/labeled_prdr_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ECON Hand-Labeled Precision: 0.57\n",
      "ECON Good Phrases Precision: 0.05\n",
      "['inverse problem', 'boosted trees', 'memory management', 'principal components analysis', 'web applications']\n",
      "['left ventricle', 'irrelevant features', 'outperform existing', 'starting point', 'mid level']\n"
     ]
    }
   ],
   "source": [
    "labeled_econ_samples = pickle.load(open('data/labeled_econ_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in labeled_econ_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'ECON Hand-Labeled Precision: {num_hl_positive / len(labeled_econ_samples)}')\n",
    "print(f'ECON Good Phrases Precision: {num_gp_positive / len(labeled_econ_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoPhrase Hand-Labeled Precision: 0.55\n",
      "AutoPhrase Good Phrases Precision: 0.03\n",
      "['motion planning', 'predictive power', 'theoretical foundation', 'shared memory', 'handwriting recognition']\n",
      "['brain imaging', 'i d', 'level features', 'road rash', 'labeled and unlabeled']\n"
     ]
    }
   ],
   "source": [
    "labeled_autophrase_samples = pickle.load(open('data/labeled_autophrase_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in labeled_autophrase_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'AutoPhrase Hand-Labeled Precision: {num_hl_positive / len(labeled_autophrase_samples)}')\n",
    "print(f'AutoPhrase Good Phrases Precision: {num_gp_positive / len(labeled_autophrase_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRDR Hand-Labeled Precision: 0.65\n",
      "PRDR Good Phrases Precision: 0.11\n",
      "['recursive matching structure', 'mnist', 'high level abstractions', 'methods', 'domain']\n",
      "['different network architectures', 'recent output representations', 'stochastic loss selection', 'significant improvement', 'neural network sentence']\n"
     ]
    }
   ],
   "source": [
    "labeled_prdr_samples = pickle.load(open('data/labeled_prdr_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in labeled_prdr_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'PRDR Hand-Labeled Precision: {num_hl_positive / len(labeled_prdr_samples)}')\n",
    "print(f'PRDR Good Phrases Precision: {num_gp_positive / len(labeled_prdr_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_samples = labeled_econ_samples + labeled_autophrase_samples + labeled_prdr_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hand-Labeled Good but not Ground Truth\n",
      "\n",
      "open set recognition\n",
      "smooth function\n",
      "boosted trees\n",
      "geometric brownian motion\n",
      "probabilistic program\n",
      "estimation error\n",
      "cifar 100\n",
      "click through rate\n",
      "standard deviation\n",
      "long term memory\n",
      "virtual worlds\n",
      "nlp applications\n",
      "group lasso\n",
      "minimum spanning tree\n",
      "disentangled representations\n",
      "inverse problem\n",
      "sampling distribution\n",
      "markov random field\n",
      "null hypothesis\n",
      "joint probability\n",
      "mixture components\n",
      "performance gain\n",
      "kernel approximation\n",
      "deep reinforcement learning rl\n",
      "functional analysis\n",
      "restricted boltzmann machine\n",
      "recognition tasks\n",
      "novelty detection\n",
      "potts model\n",
      "predictive analytics\n",
      "web browsers\n",
      "principal components analysis\n",
      "disjoint sets\n",
      "training instances\n",
      "cutting plane\n",
      "pattern mining\n",
      "weak classifiers\n",
      "deep q networks\n",
      "training phase\n",
      "hyperbolic space\n",
      "target dataset\n",
      "symbolic computation\n",
      "image datasets\n",
      "configuration space\n",
      "computational overhead\n",
      "submodular optimization\n",
      "control problems\n",
      "web applications\n",
      "moment matching\n",
      "np hard\n",
      "imagenet classification\n",
      "graph database\n",
      "pure data\n",
      "base classifier\n",
      "deep recurrent neural networks\n",
      "unified framework\n",
      "labeled dataset\n",
      "event related potential\n",
      "batch normalization\n",
      "out of order execution\n",
      "text descriptions\n",
      "optimal policy\n",
      "baseline method\n",
      "statistical significance\n",
      "statistical power\n",
      "theoretical foundation\n",
      "phonetic transcription\n",
      "multitask learning\n",
      "dynamic time warping dtw\n",
      "markov chain monte carlo\n",
      "signal strength\n",
      "toy model\n",
      "positive class\n",
      "motion planning\n",
      "outlier detection\n",
      "recommendation systems\n",
      "variational principle\n",
      "generalization capability\n",
      "handwriting recognition\n",
      "joint distributions\n",
      "classification results\n",
      "standard library\n",
      "linear representation\n",
      "virtual environment\n",
      "output units\n",
      "likelihood function\n",
      "smooth function\n",
      "expert demonstrations\n",
      "multi armed bandits\n",
      "classifier performance\n",
      "fitness landscape\n",
      "gaussian kernels\n",
      "predictive power\n",
      "expression recognition\n",
      "riemannian manifold\n",
      "observational data\n",
      "larger datasets\n",
      "proximity sensor\n",
      "fiber bundle\n",
      "collaborative filtering\n",
      "expectation propagation\n",
      "p adic number\n",
      "network layers\n",
      "tikhonov regularization\n",
      "visual object recognition\n",
      "supervised learning algorithm\n",
      "mnist\n",
      "network classifiers\n",
      "anti malware system\n",
      "neural network approach\n",
      "convolutional layers\n",
      "binary classification problem\n",
      "hierarchical reinforcement learning\n",
      "standard multitask learning\n",
      "relation classification\n",
      "neural encoder decoder\n",
      "reinforcement\n",
      "automatic evaluation\n",
      "databases\n",
      "embeddings\n",
      "multi scale representation\n",
      "batch normalization\n",
      "high level abstractions\n",
      "extreme learning\n",
      "neural transfer function\n",
      "computational convergence rates\n",
      "hierarchical value functions\n",
      "deep learning models\n",
      "test time\n",
      "progressive learning technique\n",
      "object detection\n",
      "generative models\n",
      "visual feature space\n",
      "long range dependencies\n",
      "adversarial examples\n",
      "feature engineering\n",
      "deep learning architecture\n",
      "image recognition\n",
      "recursive matching structure\n",
      "gesture recognition\n",
      "digit recognition\n",
      "visual dialog systems\n",
      "latent variables\n",
      "recurrent networks\n",
      "hierarchical memory network\n",
      "dnn\n",
      "hierarchical latent variables\n",
      "training algorithms\n",
      "input text\n",
      "speech signals\n",
      "open domain\n",
      "query\n",
      "image classifiers\n",
      "structured data\n",
      "sequence learning\n",
      "neural language models\n",
      "deep learning tasks\n",
      "gradient descent\n",
      "detection\n",
      "error rates\n",
      "visual analysis tool\n"
     ]
    }
   ],
   "source": [
    "print('Hand-Labeled Good but not Ground Truth\\n')\n",
    "for ls in labeled_samples:\n",
    "    if ls[1] == '1' and ls[0] not in good_phrases:\n",
    "        print(ls[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground Truth but not Hand-Labeled Good\n",
      "\n",
      "learning\n",
      "methods\n",
      "task\n"
     ]
    }
   ],
   "source": [
    "print('Ground Truth but not Hand-Labeled Good\\n')\n",
    "for ls in labeled_samples:\n",
    "    if ls[1] == '0' and ls[0] in good_phrases:\n",
    "        print(ls[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Phraseness Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_econ_samples = pickle.load(open('data/labeled_econ_samples.pkl', 'rb'))\n",
    "labeled_autophrase_samples = pickle.load(open('data/labeled_autophrase_samples.pkl', 'rb'))\n",
    "labeled_prdr_samples = pickle.load(open('data/labeled_prdr_samples.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "highly dependent:  0\n",
      "theoretical and empirical:  0\n",
      "sparsity inducing:  0\n",
      "moving objects:  1\n",
      "rank tensor:  0\n",
      "promising results:  1\n",
      "continuous action:  1\n",
      "recently gained:  0\n",
      "structural constraints:  1\n",
      "crop yield:  1\n",
      "optimal regret:  1\n",
      "outperform existing:  0\n",
      "malignant tumor:  1\n",
      "previous approaches:  1\n",
      "positive and negative:  0\n",
      "multiple myeloma:  1\n",
      "count based:  0\n",
      "task oriented:  0\n",
      "mid level:  0\n",
      "at large:  0\n",
      "gamma ray:  1\n",
      "applications of artificial intelligence:  1\n",
      "theoretically analyze:  0\n",
      "number of arms:  0\n",
      "gas turbine:  1\n",
      "based regularization:  0\n",
      "vending machines:  1\n",
      "exponentially large:  0\n",
      "few shot:  0\n",
      "sufficient statistics:  1\n",
      "irrelevant features:  1\n",
      "traditional medicine:  1\n",
      "set theoretic:  0\n",
      "competitive inhibition:  1\n",
      "image level:  0\n",
      "starting point:  1\n",
      "gray matter:  1\n",
      "pablo picasso:  1\n",
      "lancaster university:  1\n",
      "a long standing:  0\n",
      "spin glass:  1\n",
      "flow cytometry:  1\n",
      "left ventricle:  1\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_econ_samples = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for sample, l in labeled_econ_samples:\n",
    "    if l != '1':\n",
    "        label = input(f'{sample}: ')\n",
    "        phraseness_labeled_econ_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "    if sample in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "pickle.dump(phraseness_labeled_econ_samples, open('data/phraseness_labeled_econ_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "potential applications:  1\n",
      "stop sign:  1\n",
      "in grid:  0\n",
      "university of california irvine:  1\n",
      "robust principal component:  1\n",
      "hmm based:  0\n",
      "fiber optic:  0\n",
      "increasing attention:  0\n",
      "rare disease:  1\n",
      "empirically validate:  0\n",
      "chemical process:  1\n",
      "character level:  0\n",
      "single nucleotide polymorphisms:  1\n",
      "breaking news:  1\n",
      "lung cancer:  1\n",
      "group sparse:  0\n",
      "task of classifying:  0\n",
      "brain imaging:  1\n",
      "local planning:  1\n",
      "explicit knowledge:  1\n",
      "level features:  0\n",
      "excess risk:  1\n",
      "primary care:  1\n",
      "invasive species:  1\n",
      "the past decade:  1\n",
      "labeled and unlabeled:  0\n",
      "frame prediction:  1\n",
      "task specific:  0\n",
      "small change:  1\n",
      "few shot:  0\n",
      "traffic congestion:  1\n",
      "chemical species:  1\n",
      "i d:  0\n",
      "additional assumptions:  1\n",
      "cost sensitive:  0\n",
      "national park:  1\n",
      "road rash:  1\n",
      "powerful tools:  1\n",
      "natural selection:  1\n",
      "multiple classifiers:  11\n",
      "vulnerable to adversarial examples:  0\n",
      "human perception:  1\n",
      "mathcal o:  0\n",
      "margin based:  0\n",
      "closely related:  0\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_autophrase_samples = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for sample, l in labeled_autophrase_samples:\n",
    "    if l != '1':\n",
    "        label = input(f'{sample}: ')\n",
    "        phraseness_labeled_autophrase_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "    if sample in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "pickle.dump(phraseness_labeled_autophrase_samples, open('data/phraseness_labeled_autophrase_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "challenging reinforcement learning:  1\n",
      "structured output problems:  1\n",
      "single input image:  1\n",
      "most preprocessing steps:  1\n",
      "discrete speech code:  1\n",
      "simple similarity classifier:  1\n",
      "commercial web search:  1\n",
      "unique minimizer b:  0\n",
      "character level:  0\n",
      "high level concepts:  1\n",
      "method:  1\n",
      "learning:  1\n",
      "methods:  1\n",
      "real user experience:  1\n",
      "physical hexapod robot:  1\n",
      "many perception tasks:  1\n",
      "multi label:  0\n",
      "user:  1\n",
      "significant improvement:  1\n",
      "higher level:  0\n",
      "task:  1\n",
      "different network architectures:  1\n",
      "word level:  0\n",
      "human evaluation study:  1\n",
      "non convex:  0\n",
      "multiple image datasets:  1\n",
      "hierarchical recurrent sequence:  1\n",
      "neural machine:  1\n",
      "available training data:  1\n",
      "hard matching problem:  1\n",
      "recent output representations:  1\n",
      "new variance reduction:  1\n",
      "neural network sentence:  1\n",
      "stochastic loss selection:  1\n",
      "higher level inference:  1\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_prdr_samples = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for sample, l in labeled_prdr_samples:\n",
    "    if l != '1':\n",
    "        label = input(f'{sample}: ')\n",
    "        phraseness_labeled_prdr_samples.append((sample, label))\n",
    "        if label == '1':\n",
    "            num_hl_positive += 1\n",
    "    if sample in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "pickle.dump(phraseness_labeled_prdr_samples, open('data/phraseness_labeled_prdr_samples.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ECON Hand-Labeled Precision: 0.5581395348837209\n",
      "ECON Good Phrases Precision: 0.0\n",
      "['structural constraints', 'previous approaches', 'competitive inhibition', 'multiple myeloma', 'lancaster university']\n",
      "['rank tensor', 'positive and negative', 'at large', 'recently gained', 'task oriented']\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_econ_samples = pickle.load(open('data/phraseness_labeled_econ_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in phraseness_labeled_econ_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'ECON Hand-Labeled Precision: {num_hl_positive / len(phraseness_labeled_econ_samples)}')\n",
    "print(f'ECON Good Phrases Precision: {num_gp_positive / len(phraseness_labeled_econ_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ECON Hand-Labeled Precision: 0.81\n",
      "ECON Good Phrases Precision: 0.05\n"
     ]
    }
   ],
   "source": [
    "print(f'ECON Hand-Labeled Precision: {(num_hl_positive + 57) / 100}')\n",
    "print(f'ECON Good Phrases Precision: {(num_gp_positive + 5) / 100}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoPhrase Hand-Labeled Precision: 0.5777777777777777\n",
      "AutoPhrase Good Phrases Precision: 0.0\n",
      "['excess risk', 'chemical species', 'rare disease', 'frame prediction', 'traffic congestion']\n",
      "['multiple classifiers', 'mathcal o', 'closely related', 'margin based', 'increasing attention']\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_autophrase_samples = pickle.load(open('data/phraseness_labeled_autophrase_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in phraseness_labeled_autophrase_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'AutoPhrase Hand-Labeled Precision: {num_hl_positive / len(phraseness_labeled_autophrase_samples)}')\n",
    "print(f'AutoPhrase Good Phrases Precision: {num_gp_positive / len(phraseness_labeled_autophrase_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoPhrase Hand-Labeled Precision: 0.81\n",
      "AutoPhrase Good Phrases Precision: 0.03\n"
     ]
    }
   ],
   "source": [
    "print(f'AutoPhrase Hand-Labeled Precision: {(num_hl_positive + 55) / 100}')\n",
    "print(f'AutoPhrase Good Phrases Precision: {(num_gp_positive + 3) / 100}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRDR Hand-Labeled Precision: 0.8285714285714286\n",
      "PRDR Good Phrases Precision: 0.08571428571428572\n",
      "['high level concepts', 'neural network sentence', 'many perception tasks', 'physical hexapod robot', 'learning']\n",
      "['multi label', 'higher level', 'character level', 'unique minimizer b', 'word level']\n"
     ]
    }
   ],
   "source": [
    "phraseness_labeled_prdr_samples = pickle.load(open('data/phraseness_labeled_prdr_samples.pkl', 'rb'))\n",
    "positive = []\n",
    "negative = []\n",
    "num_hl_positive = 0\n",
    "num_gp_positive = 0\n",
    "for ls in phraseness_labeled_prdr_samples:\n",
    "    if ls[1] == '1' or ls[0] in good_phrases:\n",
    "        positive.append(ls[0])\n",
    "    else:\n",
    "        negative.append(ls[0])\n",
    "    if ls[1] == '1':\n",
    "        num_hl_positive += 1\n",
    "    if ls[0] in good_phrases:\n",
    "        num_gp_positive += 1\n",
    "print(f'PRDR Hand-Labeled Precision: {num_hl_positive / len(phraseness_labeled_prdr_samples)}')\n",
    "print(f'PRDR Good Phrases Precision: {num_gp_positive / len(phraseness_labeled_prdr_samples)}')\n",
    "print(random.sample(positive, 5))\n",
    "print(random.sample(negative, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRDR Hand-Labeled Precision: 0.94\n",
      "PRDR Good Phrases Precision: 0.14\n"
     ]
    }
   ],
   "source": [
    "print(f'PRDR Hand-Labeled Precision: {(num_hl_positive + 65) / 100}')\n",
    "print(f'PRDR Good Phrases Precision: {(num_gp_positive + 11) / 100}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}