notebooks/tmdb-tv-scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from tqdm.notebook import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "API_KEY = \"4f7e050b4d9ee8f88f4b60cc46ec39b3\"\n",
    "HEB = 'he-IL'\n",
    "ENG = 'en-US'\n",
    "EARLIEST_YEAR = 1984\n",
    "MAX_MOVIES = 2974\n",
    "MOVIES_PER_PAGE = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download top voted TV Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_rated_request = 'https://api.themoviedb.org/3/tv/top_rated?api_key={api_key}&language={lang}&page={page}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d146a8e911094c33b236a3560cf44432",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading TV Series:   0%|          | 0/148 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "top_rated_tvs = []\n",
    "\n",
    "for page in tqdm(range(1, MAX_MOVIES // 20 +1), \"Downloading TV Series\"):\n",
    "    request_url = top_rated_request.format(api_key=API_KEY, lang=HEB, page=page)\n",
    "    response = requests.get(request_url)\n",
    "    if response.status_code == 200:\n",
    "        json_data = response.json()\n",
    "        top_rated_tvs += json_data['results']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dff1c9cd041e412eb812485a85291ed6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading details and cast:   0%|          | 0/2960 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "def download_tv_data(movie_id):\n",
    "    details_url = f'https://api.themoviedb.org/3/tv/{movie_id}'\n",
    "    credits_url = f'https://api.themoviedb.org/3/tv/{movie_id}/credits'\n",
    "    keywords_url = f'https://api.themoviedb.org/3/tv/{movie_id}/keywords'\n",
    "    params = {'api_key': API_KEY}\n",
    "\n",
    "    movie_data = {}\n",
    "\n",
    "    # Downloading movie details\n",
    "    response = requests.get(details_url, params=params)\n",
    "    if response.status_code == 200:\n",
    "        movie_data['details'] = response.json()\n",
    "\n",
    "    # Downloading cast data\n",
    "    response = requests.get(credits_url, params=params)\n",
    "    if response.status_code == 200:\n",
    "        movie_data['credits'] = response.json()\n",
    "\n",
    "    # Downloading cast data\n",
    "    response = requests.get(keywords_url, params=params)\n",
    "    if response.status_code == 200:\n",
    "        movie_data['keywords'] = response.json()\n",
    "\n",
    "    return movie_id, movie_data\n",
    "\n",
    "movie_ids = [movie['id'] for movie in top_rated_tvs]\n",
    "movie_details_map = defaultdict(dict)\n",
    "\n",
    "with ThreadPoolExecutor() as executor:\n",
    "    for movie_id, movie_data in tqdm(executor.map(download_tv_data, movie_ids), total=len(movie_ids), desc=\"Downloading details and cast\"):\n",
    "        movie_details_map[movie_id] = movie_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'details': {'adult': False,\n",
       "  'backdrop_path': '/ylOY2q6pW5OV5kTyoxcuuFY5Xy4.jpg',\n",
       "  'created_by': [],\n",
       "  'episode_run_time': [43],\n",
       "  'first_air_date': '2020-01-07',\n",
       "  'genres': [{'id': 18, 'name': 'Drama'}],\n",
       "  'homepage': 'https://www.qub.ca/tvaplus/tva/epidemie',\n",
       "  'id': 95515,\n",
       "  'in_production': False,\n",
       "  'languages': ['fr'],\n",
       "  'last_air_date': '2020-03-10',\n",
       "  'last_episode_to_air': {'id': 2163499,\n",
       "   'name': 'Scars',\n",
       "   'overview': 'Two days before the official announcement that the epidemic is over, the infection of two little girls brings Anne-Marie back to square one. The epidemiological investigation resumes, but the girls are uncooperative. Will Nelli get them to talk and prevent COVA from claiming more victims?',\n",
       "   'vote_average': 0.0,\n",
       "   'vote_count': 0,\n",
       "   'air_date': '2020-03-10',\n",
       "   'episode_number': 10,\n",
       "   'episode_type': 'finale',\n",
       "   'production_code': '',\n",
       "   'runtime': 43,\n",
       "   'season_number': 1,\n",
       "   'show_id': 95515,\n",
       "   'still_path': '/7xOizJ0zxoJvSHHQHj2e2NaPiqT.jpg'},\n",
       "  'name': 'Outbreak',\n",
       "  'next_episode_to_air': None,\n",
       "  'networks': [{'id': 302,\n",
       "    'logo_path': '/gc3SsJUQa7mGnHLds13zzoxG3zd.png',\n",
       "    'name': 'TVA',\n",
       "    'origin_country': 'CA'}],\n",
       "  'number_of_episodes': 10,\n",
       "  'number_of_seasons': 1,\n",
       "  'origin_country': ['CA'],\n",
       "  'original_language': 'fr',\n",
       "  'original_name': 'Épidémie',\n",
       "  'overview': 'A dangerous virus appears in a group of homeless people, causing a risk of outbreak. How long will it take Anne-Marie Leclerc, director of the Emergency Public Health Laboratory, to realize that an actual epidemic is starting to take shape?',\n",
       "  'popularity': 35.767,\n",
       "  'poster_path': '/dwfYZrGs898pSfMU3IxR4fim3Nu.jpg',\n",
       "  'production_companies': [{'id': 169682,\n",
       "    'logo_path': '/dSajIaySHCYM1jsxFVg7lypSATH.png',\n",
       "    'name': 'Sphère Média',\n",
       "    'origin_country': 'CA'}],\n",
       "  'production_countries': [{'iso_3166_1': 'CA', 'name': 'Canada'}],\n",
       "  'seasons': [{'air_date': '2020-01-07',\n",
       "    'episode_count': 10,\n",
       "    'id': 135959,\n",
       "    'name': 'Season 1',\n",
       "    'overview': '',\n",
       "    'poster_path': '/8UC83gyT6BG4Cm4lSqrzddc228f.jpg',\n",
       "    'season_number': 1,\n",
       "    'vote_average': 0.0}],\n",
       "  'spoken_languages': [{'english_name': 'French',\n",
       "    'iso_639_1': 'fr',\n",
       "    'name': 'Français'}],\n",
       "  'status': 'Ended',\n",
       "  'tagline': '',\n",
       "  'type': 'Scripted',\n",
       "  'vote_average': 7.286,\n",
       "  'vote_count': 126},\n",
       " 'credits': {'cast': [{'adult': False,\n",
       "    'gender': 1,\n",
       "    'id': 142996,\n",
       "    'known_for_department': 'Acting',\n",
       "    'name': 'Julie Le Breton',\n",
       "    'original_name': 'Julie Le Breton',\n",
       "    'popularity': 4.126,\n",
       "    'profile_path': '/pMSYAWIYfdEWlLU8dHejp93mRIy.jpg',\n",
       "    'character': 'Anne-Marie Lerclerc',\n",
       "    'credit_id': '5dd0be4f2d531a0018d8d407',\n",
       "    'order': 3}],\n",
       "  'crew': [{'adult': False,\n",
       "    'gender': 2,\n",
       "    'id': 1294685,\n",
       "    'known_for_department': 'Writing',\n",
       "    'name': 'Bernard Dansereau',\n",
       "    'original_name': 'Bernard Dansereau',\n",
       "    'popularity': 0.6,\n",
       "    'profile_path': None,\n",
       "    'credit_id': '5dd0bea0b76cbb00167a2e75',\n",
       "    'department': 'Writing',\n",
       "    'job': 'Writer'},\n",
       "   {'adult': False,\n",
       "    'gender': 1,\n",
       "    'id': 1294686,\n",
       "    'known_for_department': 'Writing',\n",
       "    'name': 'Annie Piérard',\n",
       "    'original_name': 'Annie Piérard',\n",
       "    'popularity': 0.6,\n",
       "    'profile_path': None,\n",
       "    'credit_id': '5dd0be992d531a0015d904dc',\n",
       "    'department': 'Writing',\n",
       "    'job': 'Writer'},\n",
       "   {'adult': False,\n",
       "    'gender': 0,\n",
       "    'id': 1157120,\n",
       "    'known_for_department': 'Writing',\n",
       "    'name': 'Yan Lanouette Turgeon',\n",
       "    'original_name': 'Yan Lanouette Turgeon',\n",
       "    'popularity': 1.4,\n",
       "    'profile_path': None,\n",
       "    'credit_id': '5dd0be8e2d531a0018d8d44c',\n",
       "    'department': 'Directing',\n",
       "    'job': 'Director'}],\n",
       "  'id': 95515},\n",
       " 'keywords': {'id': 95515,\n",
       "  'results': [{'name': 'montreal, canada', 'id': 6719},\n",
       "   {'name': 'epidemic', 'id': 17995},\n",
       "   {'name': 'virus', 'id': 188957}]}}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_details_map[95515]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compose union json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keys(entry, keys):\n",
    "    return {key: entry[key] for key in keys if key in entry}\n",
    "\n",
    "def extract_list_be_key(entries, key):\n",
    "    return [entry[key] for entry in entries]\n",
    "\n",
    "\n",
    "def populate_movie(movie, details, credits, keywords):\n",
    "    movie['genres'] = extract_list_be_key(details['genres'], 'name')\n",
    "    movie['name_he'] = movie['name']\n",
    "    movie['name'] = details['name']\n",
    "    movie['year'] = details['first_air_date'].split('-')[0]\n",
    "    \n",
    "    movie['overview_he'] = movie['overview']\n",
    "    movie['overview'] = details['overview']\n",
    "  \n",
    "    movie['production_companies'] = extract_list_be_key(details['production_companies'], 'name')\n",
    "    movie['networks'] = extract_list_be_key(details['production_companies'], 'name')\n",
    "\n",
    "    movie['cast'] = [extract_keys(p, ['name', 'character', 'profile_path','order']) for p in credits['cast'][:8]]\n",
    "    movie['creators'] =  extract_list_be_key(details['created_by'], 'name')\n",
    "\n",
    "    movie['keywords'] = extract_list_be_key(keywords, 'name')\n",
    "\n",
    "    if 'genre_ids' in movie:\n",
    "        del movie['genre_ids']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for movie in top_rated_tvs:\n",
    "    id = movie['id']\n",
    "    populate_movie(movie, movie_details_map[id]['details'], movie_details_map[id]['credits'], movie_details_map[id]['keywords']['results'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'backdrop_path': '/1QTP2ZIiT6oIrlZvZcCILA86Op.jpg',\n",
       " 'first_air_date': '2020-01-07',\n",
       " 'id': 95515,\n",
       " 'name': 'Outbreak',\n",
       " 'origin_country': ['CA'],\n",
       " 'original_language': 'fr',\n",
       " 'original_name': 'Épidémie',\n",
       " 'overview': 'A dangerous virus appears in a group of homeless people, causing a risk of outbreak. How long will it take Anne-Marie Leclerc, director of the Emergency Public Health Laboratory, to realize that an actual epidemic is starting to take shape?',\n",
       " 'popularity': 35.767,\n",
       " 'poster_path': '/aYHqgip3sct7nHw5AdLV4PYJzG6.jpg',\n",
       " 'vote_average': 7.3,\n",
       " 'vote_count': 126,\n",
       " 'genres': ['Drama'],\n",
       " 'name_he': 'Épidémie',\n",
       " 'year': '2020',\n",
       " 'overview_he': '',\n",
       " 'production_companies': ['Sphère Média'],\n",
       " 'networks': ['Sphère Média'],\n",
       " 'cast': [{'name': 'Julie Le Breton',\n",
       "   'character': 'Anne-Marie Lerclerc',\n",
       "   'profile_path': '/pMSYAWIYfdEWlLU8dHejp93mRIy.jpg',\n",
       "   'order': 3}],\n",
       " 'creators': [],\n",
       " 'keywords': ['montreal, canada', 'epidemic', 'virus']}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_rated_tvs[2224]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export json and CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('top-voted-tvs.json', 'w', encoding='utf-8') as f1:\n",
    "    json.dump(top_rated_tvs, f1, ensure_ascii=False)\n",
    "\n",
    "with open('tvs-map.json', 'w', encoding='utf-8') as f2:\n",
    "    tvs_map = {m['id']: m for m in top_rated_tvs}\n",
    "    json.dump(tvs_map, f2, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame.from_dict(top_rated_tvs)\n",
    "df.drop(columns=['name_he', 'overview_he', 'original_name'], inplace=True)\n",
    "df.to_csv('top_voted_tvs.csv', encoding='utf-8', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "notebooks_env",
   "language": "python",
   "name": "notebooks_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "dea6d499cbf397c26ae013ea8e53ba5411bcbd1f3ca809ab02d0d1b7bcfb1cbc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}