CPF_RG_CNPJ_CEP_SpaCy.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNvWB9pdMEdg23PPrcq3h4y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/felipecasali-usp/mba-tcc-identify-lgpdsensitive-data/blob/main/CPF_RG_CNPJ_CEP_SpaCy.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "hDpHJjtGHOvp"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "import spacy\n",
        "from spacy.training.example import Example\n",
        "\n",
        "# Load the CSV file into a Pandas DataFrame\n",
        "df = pd.read_csv('/content/sample_data/empregados_m.csv')\n",
        "\n",
        "# Define regular expressions to identify CPF, RG, CNPJ, and CEP numbers\n",
        "cpf_regex = r'\\b\\d{3}\\.\\d{3}\\.\\d{3}-\\d{2}\\b'\n",
        "rg_regex = r'\\b\\d{2}\\.\\d{3}\\.\\d{3}-\\d{1}\\b'\n",
        "cnpj_regex = r'\\b\\d{2}\\.\\d{3}\\.\\d{3}/\\d{4}-\\d{2}\\b'\n",
        "cep_regex = r'\\b\\d{5}-\\d{3}\\b'\n",
        "\n",
        "# Prepare training data with examples of annotated text\n",
        "TRAIN_DATA = []\n",
        "\n",
        "# Iterate through each row of the DataFrame\n",
        "for index, row in df.iterrows():\n",
        "    # Concatenate all values in the row into a single string\n",
        "    row_text = ' '.join(map(str, row.values))\n",
        "\n",
        "    # Search for CPF numbers using the regular expression\n",
        "    cpf_matches = re.findall(cpf_regex, row_text)\n",
        "    rg_matches = re.findall(rg_regex, row_text)\n",
        "    cnpj_matches = re.findall(cnpj_regex, row_text)\n",
        "    cep_matches = re.findall(cep_regex, row_text)\n",
        "\n",
        "    # Annotate the position of CPF, RG, CNPJ, and CEP numbers\n",
        "    entities = []\n",
        "    for match in cpf_matches:\n",
        "        start = row_text.find(match)\n",
        "        end = start + len(match)\n",
        "        entities.append((start, end, \"CPF\"))\n",
        "    for match in rg_matches:\n",
        "        start = row_text.find(match)\n",
        "        end = start + len(match)\n",
        "        entities.append((start, end, \"RG\"))\n",
        "    for match in cnpj_matches:\n",
        "        start = row_text.find(match)\n",
        "        end = start + len(match)\n",
        "        entities.append((start, end, \"CNPJ\"))\n",
        "    for match in cep_matches:\n",
        "        start = row_text.find(match)\n",
        "        end = start + len(match)\n",
        "        entities.append((start, end, \"CEP\"))\n",
        "\n",
        "    # Add the annotated example to the training data\n",
        "    TRAIN_DATA.append((row_text, {\"entities\": entities}))\n",
        "\n",
        "# Output the annotated data\n",
        "#print(TRAIN_DATA)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "import random\n",
        "from spacy.training.example import Example\n",
        "\n",
        "# Load a blank English model\n",
        "nlp = spacy.blank(\"en\")\n",
        "\n",
        "# Add the NER pipeline component\n",
        "ner = nlp.add_pipe(\"ner\")\n",
        "\n",
        "# Define your entity label\n",
        "ner.add_label(\"SSN\")\n",
        "\n",
        "# Start the training process\n",
        "nlp.begin_training()\n",
        "\n",
        "# Train for a number of iterations\n",
        "for itn in range(20):\n",
        "    # Shuffle the training data\n",
        "    random.shuffle(TRAIN_DATA)\n",
        "    losses = {}\n",
        "\n",
        "    # Create batches and train the model\n",
        "    for batch in spacy.util.minibatch(TRAIN_DATA, size=2):\n",
        "        texts, annotations = zip(*batch)\n",
        "        example = []\n",
        "        # Update the model with the current batch\n",
        "        for i in range(len(texts)):\n",
        "            doc = nlp.make_doc(texts[i])\n",
        "            example.append(Example.from_dict(doc, annotations[i]))\n",
        "        nlp.update(example, losses=losses)\n",
        "    print(\"Iteration:\", itn, \"Losses:\", losses)\n",
        "\n",
        "# Save the trained model\n",
        "nlp.to_disk(\"ssn_ner_model\")"
      ],
      "metadata": {
        "id": "w8OTeCqaI5Os"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "import random\n",
        "from spacy.training.example import Example\n",
        "\n",
        "# Load a blank English model\n",
        "nlp = spacy.blank(\"en\")\n",
        "\n",
        "# Add the NER pipeline component\n",
        "ner = nlp.add_pipe(\"ner\")\n",
        "\n",
        "# Define your entity labels\n",
        "labels = [\"CPF\", \"RG\", \"CNPJ\", \"CEP\"]\n",
        "\n",
        "# Add the labels to the NER component\n",
        "for label in labels:\n",
        "    ner.add_label(label)\n",
        "\n",
        "# Start the training process\n",
        "nlp.begin_training()\n",
        "\n",
        "# Train for a number of iterations\n",
        "for itn in range(14):\n",
        "    # Shuffle the training data\n",
        "    random.shuffle(TRAIN_DATA)\n",
        "    losses = {}\n",
        "\n",
        "    # Create batches and train the model\n",
        "    for batch in spacy.util.minibatch(TRAIN_DATA, size=64):\n",
        "        texts, annotations = zip(*batch)\n",
        "        example = []\n",
        "        # Update the model with the current batch\n",
        "        for i in range(len(texts)):\n",
        "            doc = nlp.make_doc(texts[i])\n",
        "            example.append(Example.from_dict(doc, annotations[i]))\n",
        "        nlp.update(example, losses=losses)\n",
        "    print(\"Iteration:\", itn, \"Losses:\", losses)\n",
        "\n",
        "# Save the trained model\n",
        "nlp.to_disk(\"ssn_ner_model\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YqUN-GM5sh1S",
        "outputId": "26b19a54-5030-4f4b-b2e8-8aaa65800432"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration: 0 Losses: {'ner': 38000.63148630018}\n",
            "Iteration: 1 Losses: {'ner': 2708.761159885138}\n",
            "Iteration: 2 Losses: {'ner': 1711.3176548783476}\n",
            "Iteration: 3 Losses: {'ner': 681.7381961791214}\n",
            "Iteration: 4 Losses: {'ner': 442.19583946377134}\n",
            "Iteration: 5 Losses: {'ner': 345.4126415976802}\n",
            "Iteration: 6 Losses: {'ner': 349.7324265278233}\n",
            "Iteration: 7 Losses: {'ner': 270.638975614015}\n",
            "Iteration: 8 Losses: {'ner': 210.69144856355544}\n",
            "Iteration: 9 Losses: {'ner': 246.25540855076972}\n",
            "Iteration: 10 Losses: {'ner': 248.92106825233557}\n",
            "Iteration: 11 Losses: {'ner': 162.56041760766294}\n",
            "Iteration: 12 Losses: {'ner': 141.69182367444122}\n",
            "Iteration: 13 Losses: {'ner': 138.31533491108877}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "# Load the saved model\n",
        "nlp = spacy.load(\"ssn_ner_model\")\n",
        "\n",
        "# Example text for testing\n",
        "test_text = \"O meu número de CPF é 282.503.128-30\"\n",
        "\n",
        "# Process the text using the loaded model\n",
        "doc = nlp(test_text)\n",
        "\n",
        "# Print detected entities\n",
        "for ent in doc.ents:\n",
        "    print(ent.text, ent.label_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2AwO2zGDkb5",
        "outputId": "8104013c-4f48-4048-cb97-2322dacafff6"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "282.503.128-30 CPF\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import spacy\n",
        "\n",
        "# Load the saved model\n",
        "nlp = spacy.load(\"ssn_ner_model\")\n",
        "\n",
        "# List of test sentences\n",
        "test_sentences = [\n",
        "    \"Felipe Casali's identification number is 297.222.108-09\",\n",
        "    \"Her wife document is 282.555.128-30\",\n",
        "    \"His RG is 30.746.719-3\",\n",
        "    \"His RG is 307467193\",\n",
        "    \"Felipe mora no 05092-020\"\n",
        "]\n",
        "\n",
        "# Process each test sentence using the loaded model\n",
        "for test_sentence in test_sentences:\n",
        "    doc = nlp(test_sentence)\n",
        "    print(\"Test Sentence:\", test_sentence)\n",
        "    if not doc.ents:\n",
        "        print(\"No entities found.\")\n",
        "    else:\n",
        "        # Print detected entities\n",
        "        for ent in doc.ents:\n",
        "            print(ent.text, ent.label_)\n",
        "    print()  # Empty line for readability between sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wB8AqQIADwGT",
        "outputId": "1581458d-361a-4482-f2af-2a6d9e0b76bc"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Sentence: Felipe Casali's identification number is 297.222.108-09\n",
            "297.222.108-09 CPF\n",
            "\n",
            "Test Sentence: Her wife document is 282.555.128-30\n",
            "282.555.128-30 CPF\n",
            "\n",
            "Test Sentence: His RG is 30.746.719-3\n",
            "No entities found.\n",
            "\n",
            "Test Sentence: His RG is 307467193\n",
            "No entities found.\n",
            "\n",
            "Test Sentence: Felipe mora no 05092-020\n",
            "05092-020 CEP\n",
            "\n"
          ]
        }
      ]
    }
  ]
}