Scrape Google Play Children (Kids) with Python
Tags: webscraping, tutorial, python, programming
Published: 2022-11-15 15:22:04

- <a href="#what_will_be_scraped">What will be scraped</a>
- <a href="#fullcode">Full Code</a>
- <a href="#preparation">Preparation</a>
- <a href="#code_explanation">Code Explanation</a>
    - <a href="#top_level_code_environment">Top-level code environment</a>
    - <a href="#scroll_page">Scroll page</a>
    - <a href="#scrape_all_sections">Scrape all sections</a>
    - <a href="#scrape_new_and_updated">Scrape New & Updated section</a>
    - <a href="#output">Output</a>
- <a href="#google_play_children">Using Google Play Apps Store API from SerpApi</a>
- <a href="#links">Links</a>


<h2 id="what_will_be_scraped">What will be scraped</h2>

![wwbs-google-play-children](https://user-images.githubusercontent.com/81998012/195469333-4e19ee2b-2e86-4846-85e3-07917e5159a2.png)

📌Note: Google Play gives different results for logged in and not logged in users.


<h2 id="fullcode">Full Code</h2>

If you don't need explanation, have a look at [full code example in the online IDE](https://replit.com/@chukhraiartur/blog-google-play-children#main.py).

```python
import time, json
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from parsel import Selector

google_play_children = {}


def scroll_page(url):	
    service = Service(ChromeDriverManager().install())

    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--lang=en")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")
    options.add_argument("--no-sandbox")

    driver = webdriver.Chrome(service=service, options=options)
    driver.get(url)

    old_height = driver.execute_script("""
        function getHeight() {
            return document.querySelector('.T4LgNb').scrollHeight;
        }
        return getHeight();
    """)

    while True:
        driver.execute_script("window.scrollTo(0, document.querySelector('.T4LgNb').scrollHeight)")
        time.sleep(1)

        new_height = driver.execute_script("""
            function getHeight() {
                return document.querySelector('.T4LgNb').scrollHeight;
            }
            return getHeight();
        """)

        if new_height == old_height:
            break

        old_height = new_height

    selector = Selector(driver.page_source)
    driver.quit()

    return selector


def scrape_new_and_updated(selector):
    google_play_children['New & Updated'] = []

    for item in selector.css('.TAQqTe'):
        title = item.css('.OnEJge::text').get()
        link = 'https://play.google.com' + item.css('::attr(href)').get()
        category = item.css('.ubGTjb > .w2kbF::text').getall()[-1]
        rating = item.css('.ubGTjb div .w2kbF::text').get()
        rating = float(rating) if rating else rating
        price = item.css('.ePXqnb::text').get()
        price = float(price[1:]) if price else 'free'
        thumbnail = item.css('.j2FCNc img::attr(srcset)').get().replace(' 2x', '')
        screenshot_image = item.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()
        video = item.css('.XdjT2b::attr(data-video-url)').get()
        
        google_play_children['New & Updated'].append({
            'title': title,
            'link': link,
            'category': category,
            'rating': rating,
            'price': price,
            'thumbnail': thumbnail,
            'screenshot_image': screenshot_image,
            'video': video
        })


def scrape_all_sections(selector):	
    for section in selector.css('section'):
        section_title = section.css('.kcen6d span::text').get()
        
        if section_title is None:
            continue
        elif section_title == 'New & Updated':
            scrape_new_and_updated(section)
            continue
            
        google_play_children[section_title] = []
        
        for item in section.css('.UVEnyf'):
            title = item.css('.Epkrse::text').get()
            link = 'https://play.google.com' + item.css('.Si6A0c::attr(href)').get()
            rating = item.css('.LrNMN:nth-child(1)::text').get()
            rating = float(rating) if rating else rating
            price = item.css('.VixbEe span::text').get()
            price = float(price[1:]) if price else 'free'
            thumbnail = item.css('.etjhNc::attr(srcset)').get()
            thumbnail = thumbnail.replace(' 2x', '') if thumbnail else thumbnail

            google_play_children[section_title].append({
                'title': title,
                'link': link,
                'rating': rating,
                'price': price,
                'thumbnail': thumbnail
            })

    print(json.dumps(google_play_children, indent=2, ensure_ascii=False))


def scrape_google_play_children():
    params = {
        'hl': 'en_GB',		        # language 
        'gl': 'US',	        		# country of the search
        'age': None                 # category. AGE_RANGE1 = up to 5 years, AGE_RANGE2 = 6-8 years, AGE_RANGE3 = 9-12 years
    }

    if params['age'] in ['AGE_RANGE1', 'AGE_RANGE2', 'AGE_RANGE3']:
        URL = f"https://play.google.com/store/apps/category/FAMILY?age={params['age']}&hl={params['hl']}&gl={params['gl']}"
    else:
        URL = f"https://play.google.com/store/apps/category/FAMILY?hl={params['hl']}&gl={params['gl']}"

    result = scroll_page(URL)
    scrape_all_sections(result)


if __name__ == "__main__":
    scrape_google_play_children()
```


<h2 id="preparation">Preparation</h2>

**Install libraries**:

```lang-none
pip install parsel selenium webdriver webdriver_manager
```

**Reduce the chance of being blocked**

Make sure you're using [request headers](https://docs.python-requests.org/en/master/user/quickstart/#custom-headers) [`user-agent`](https://developer.mozilla.org/en-US/docs/Glossary/User_agent) to act as a "real" user visit. Because default `requests` `user-agent` is [`python-requests`](https://github.com/psf/requests/blob/589c4547338b592b1fb77c65663d8aa6fbb7e38b/requests/utils.py#L808-L814) and websites understand that it's most likely a script that sends a request. [Check what's your `user-agent`](https://www.whatismybrowser.com/detect/what-is-my-user-agent/).

There's a [how to reduce the chance of being blocked while web scraping blog post](https://serpapi.com/blog/how-to-reduce-chance-of-being-blocked-while-web/) that can get you familiar with basic and more advanced approaches.


<h2 id="code_explanation">Code Explanation</h2>

Import libraries:

```python
import time, json
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from parsel import Selector
```

| Library       | Purpose                                                            |
|---------------|--------------------------------------------------------------------|
| [`time`](https://docs.python.org/3/library/time.html)    | to work with time in Python.                                  |
| [`json`](https://docs.python.org/3/library/json.html)        | to convert extracted data to a JSON object.                        |
| [`webdriver`](https://www.selenium.dev/documentation/webdriver/)    | to drive a browser natively, as a user would, either locally or on a remote machine using the Selenium server.                                  |
| [`Service`](https://www.selenium.dev/selenium/docs/api/py/webdriver_chrome/selenium.webdriver.chrome.service.html)    | to manage the starting and stopping of the ChromeDriver.                                  |
| [`Selector`](https://parsel.readthedocs.io/en/latest/)      | XML/HTML parser that have full [XPath](https://en.wikipedia.org/wiki/XPath) and CSS selectors support. |

Define the dictionary in which all the extracted data will be stored:

```python
google_play_children = {}
```


<h3 id="top_level_code_environment">Top-level code environment</h3>

At the beginning of the function, parameters are defined for generating the `URL`. If you want to pass other parameters to the URL, you can do so using the `params` dictionary. The parameters affect the output results:

```python
params = {
    'hl': 'en_GB',		        # language 
    'gl': 'US',	        		# country of the search
    'age': None                 # category. AGE_RANGE1 = up to 5 years, AGE_RANGE2 = 6-8 years, AGE_RANGE3 = 9-12 years
}
```

I want to draw your attention to the fact that by clicking on different categories, different links are formed. This is illustrated more clearly in the GIF below:

![google-play-children-category](https://user-images.githubusercontent.com/81998012/195470407-24e0a231-27db-4469-b911-3a9370a45574.gif)

In order for the code to work correctly with each of the categories, it was decided to create a condition according to which the corresponding link will be formed:

```python
if params['age'] in ['AGE_RANGE1', 'AGE_RANGE2', 'AGE_RANGE3']:
    URL = f"https://play.google.com/store/apps/category/FAMILY?age={params['age']}&hl={params['hl']}&gl={params['gl']}"
else:
    URL = f"https://play.google.com/store/apps/category/FAMILY?hl={params['hl']}&gl={params['gl']}"
```

|  Category   | The value of the "age" parameter |
|-------------|----------------------------------|
| __All ages up to 12__ | `None` |
| __Ages up to 5__ | `AGE_RANGE1` |
| __Ages 6-8__ | `AGE_RANGE2` |
| __Ages 9-12__ | `AGE_RANGE3` |

Next, the URL is passed to the `scroll_page(URL)` function to scroll the page and get all data. The result that this function returns is passed to the `scrape_all_sections(result)` function to extract the necessary data. The explanation of these functions will be in the corresponding headings below.

```python
result = scroll_page(URL)
scrape_all_sections(result)
```

This code uses the generally accepted rule of using the [`__name__ == "__main__"`](https://docs.python.org/3/library/__main__.html#) construct: 

```python
def scrape_google_play_children():
    params = {
        'hl': 'en_GB',		        # language 
        'gl': 'US',	        		# country of the search
        'age': None                 # category. AGE_RANGE1 = up to 5 years, AGE_RANGE2 = 6-8 years, AGE_RANGE3 = 9-12 years
    }

    if params['age'] in ['AGE_RANGE1', 'AGE_RANGE2', 'AGE_RANGE3']:
        URL = f"https://play.google.com/store/apps/category/FAMILY?age={params['age']}&hl={params['hl']}&gl={params['gl']}"
    else:
        URL = f"https://play.google.com/store/apps/category/FAMILY?hl={params['hl']}&gl={params['gl']}"

    result = scroll_page(URL)
    scrape_all_sections(result)


if __name__ == "__main__":
    scrape_google_play_children()
```

This check will only be performed if the user has run this file. If the user imports this file into another, then the check will not work.

You can watch the video [Python Tutorial: if __name__ == '__main__'](https://www.youtube.com/watch?v=sugvnHA7ElY&t=1s) for more details.


<h3 id="scroll_page">Scroll page</h3>

The function takes the URL and returns a full HTML structure.

First, let's understand how pagination works on the Google Play Children page. Data does not load immediately. If the user needs more data, they will simply scroll the page and site download a small package of data. 

In this case, [`selenium` library](https://www.selenium.dev/documentation/webdriver/) is used, which allows you to simulate user actions in the browser. For `selenium` to work, you need to use [`ChromeDriver`](https://chromedriver.chromium.org/downloads), which can be downloaded manually or using code. In our case, the second method is used. To control the start and stop of `ChromeDriver`, you need to use [`Service`](https://www.selenium.dev/selenium/docs/api/py/webdriver_chrome/selenium.webdriver.chrome.service.html) which will install browser binaries under the hood:

```python
service = Service(ChromeDriverManager().install())
```

You should also add `options` to work correctly:

```python
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--lang=en')
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36')
options.add_argument('--no-sandbox')
```

| Chrome options     | Explanation    |
|--------------------|----------------|
| `--headless` | to run Chrome in headless mode. |
| `--lang=en` | to set the browser language to English. |
| `user-agent` | to act as a "real" user request from the browser by passing it to [request headers](https://docs.python-requests.org/en/master/user/quickstart/#custom-headers). [Check what's your `user-agent`](https://www.whatismybrowser.com/detect/what-is-my-user-agent). |
| `--no-sandbox` | to make chromedriver work properly on different machines. |

Now we can start [`webdriver`](https://www.selenium.dev/documentation/webdriver/) and pass the url to the [`get()` method](https://selenium-python.readthedocs.io/api.html#selenium.webdriver.remote.webdriver.WebDriver.get).

```python
driver = webdriver.Chrome(service=service, options=options)
driver.get(url)
```

The page scrolling algorithm looks like this:

1. Find out the initial page height and write the result to the `old_height` variable.
2. Scroll the page using the script and wait 2 seconds for the data to load.
3. Find out the new page height and write the result to the `new_height` variable.
4. If the variables `new_height` and `old_height` are equal, then we complete the algorithm, otherwise we write the value of the variable `new_height` to the variable `old_height` and return to step 2.

Getting the page height and scroll is done by pasting the JavaScript code into the [`execute_script()`](https://selenium-python.readthedocs.io/api.html#selenium.webdriver.remote.webdriver.WebDriver.execute_script) method.

```python
# 1 step
old_height = driver.execute_script("""
    function getHeight() {
        return document.querySelector('.T4LgNb').scrollHeight;
    }
    return getHeight();
""")

while True:
    # 2 step
    driver.execute_script("window.scrollTo(0, document.querySelector('.T4LgNb').scrollHeight)")
    time.sleep(1)

    # 3 step
    new_height = driver.execute_script("""
        function getHeight() {
            return document.querySelector('.T4LgNb').scrollHeight;
        }
        return getHeight();
    """)

    # 4 step
    if new_height == old_height:
        break

    old_height = new_height
```

Now we need to process HTML using from [`Parsel`](https://parsel.readthedocs.io/en/latest/) package, in which we pass the `HTML` structure with all the data that was received after scrolling the page. This is necessary to successfully retrieve data in the next function. After all the operations are done, stop the driver:

```python
selector = Selector(driver.page_source)
driver.quit()
```

The function looks like this:

```python
def scroll_page(url):	
    service = Service(ChromeDriverManager().install())

    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--lang=en")
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")
    options.add_argument("--no-sandbox")

    driver = webdriver.Chrome(service=service, options=options)
    driver.get(url)

    old_height = driver.execute_script("""
        function getHeight() {
            return document.querySelector('.T4LgNb').scrollHeight;
        }
        return getHeight();
    """)

    while True:
        driver.execute_script("window.scrollTo(0, document.querySelector('.T4LgNb').scrollHeight)")
        time.sleep(1)

        new_height = driver.execute_script("""
            function getHeight() {
                return document.querySelector('.T4LgNb').scrollHeight;
            }
            return getHeight();
        """)

        if new_height == old_height:
            break

        old_height = new_height

    selector = Selector(driver.page_source)
    driver.quit()

    return selector
```

In the gif below, I demonstrate how this function works:

![google-play-children-scroll](https://user-images.githubusercontent.com/81998012/195474657-3b9908b1-5308-4f5f-aed9-8352b07f0b03.gif)


<h3 id="scrape_all_sections">Scrape all sections</h3>

This function takes a full HTML structure and prints all results in JSON format.

To retrieve data from all sections, you need to find the `section` selector of the section. You need to iterate each section in the loop:

```python
for section in selector.css('section'):
    # data extraction will be here
```

It is necessary to extract the `section_title` and make it a key in the `google_play_children` dictionary, where a list of extracted data for each item will be added later.

Pay attention to check the value of the `section_title` variable:

1. The first condition is used because sometimes an empty section with no values is retrieved. 
2. The second condition is used to retrieve data from the special section. This is the _New & Updated_ section, where data is retrieved differently and for this the `scrape_new_and_updated(section)` function is used, which will be discussed in the corresponding heading.

```python
section_title = section.css('.kcen6d span::text').get()
        
if section_title is None:
    continue
elif section_title == 'New & Updated':
    scrape_new_and_updated(section)
    continue
    
google_play_children[section_title] = []
```

To extract the necessary data, you need to find the selector where they are located. In our case, this is the `.UVEnyf` selector, which contains all items. You need to iterate each item in the loop:

```python
for item in section.css('.UVEnyf'):
    # data extraction will be here
```

For each item, data such as `title`, `link`, `rating`, `price` and `thumbnail` are extracted. You need to find the matching selector and get the text or attribute value. I want to additionally note that the `thumbnail` is retrieved from the `srcset` attribute, where it is of better quality:

```python
title = item.css('.Epkrse::text').get()
link = 'https://play.google.com' + item.css('.Si6A0c::attr(href)').get()
rating = item.css('.LrNMN:nth-child(1)::text').get()
rating = float(rating) if rating else rating
price = item.css('.VixbEe span::text').get()
price = float(price[1:]) if price else 'free'
thumbnail = item.css('.etjhNc::attr(srcset)').get()
thumbnail = thumbnail.replace(' 2x', '') if thumbnail else thumbnail
```

📌Note: When extracting the `rating`, `price` and `thumbnail`, a [ternary expression](https://docs.python.org/3/reference/expressions.html#conditional-expressions) is used which handles the values of these data, if any are available.

After extracting the data, they are appended to the `google_play_children` dictionary by the corresponding key:

```python
google_play_children[section_title].append({
    'title': title,
    'link': link,
    'rating': rating,
    'price': price,
    'thumbnail': thumbnail
})
```

At the end of the function, the `google_play_children` dictionary is printed in JSON format using the [`json.dumps()` function](https://docs.python.org/3/library/json.html#json.dumps):

```python
print(json.dumps(google_play_children, indent=2, ensure_ascii=False))
```

The complete function to scrape all sections would look like this:

```python
def scrape_all_sections(selector):	
    for section in selector.css('section'):
        section_title = section.css('.kcen6d span::text').get()
        
        if section_title is None:
            continue
        elif section_title == 'New & Updated':
            scrape_new_and_updated(section)
            continue
            
        google_play_children[section_title] = []
        
        for item in section.css('.UVEnyf'):
            title = item.css('.Epkrse::text').get()
            link = 'https://play.google.com' + item.css('.Si6A0c::attr(href)').get()
            rating = item.css('.LrNMN:nth-child(1)::text').get()
            rating = float(rating) if rating else rating
            price = item.css('.VixbEe span::text').get()
            price = float(price[1:]) if price else 'free'
            thumbnail = item.css('.etjhNc::attr(srcset)').get()
            thumbnail = thumbnail.replace(' 2x', '') if thumbnail else thumbnail

            google_play_children[section_title].append({
                'title': title,
                'link': link,
                'rating': rating,
                'price': price,
                'thumbnail': thumbnail
            })

    print(json.dumps(google_play_children, indent=2, ensure_ascii=False))
```

|Code|Explanation|
|----|-----------|
|[`css()`](https://parsel.readthedocs.io/en/latest/usage.html#using-selectors)| to access elements by the passed selector.|
|[`::text` or `::attr(<attribute>)`](https://github.com/scrapy/parsel/blob/90397dcd0b2c1cbb91e44f65c50f9e11628ba028/parsel/csstranslator.py#L48-L51)| to extract textual or attribute data from the node. |
|[`get()`](https://parsel.readthedocs.io/en/latest/usage.html#usage)| to actually extract the textual data.|
|[`float()`](https://docs.python.org/3/library/functions.html#float)| to make a floating number from a string value.|
|[`replace()`](https://docs.python.org/3/library/stdtypes.html#str.replace)| to replace all occurrences of the old substring with the new one without extra elements.|


<h3 id="scrape_new_and_updated">Scrape New & Updated section</h3>

This function accepts the _New & Updated_ section selector and writes the results to the `google_play_children` dictionary by the corresponding key.

The _New & Updated_ section does not always appear. It all depends on the passed parameters and the selected category. Accordingly, inside this function, you need to create the corresponding key in the `google_play_children` dictionary and declare a list in it:

```python
google_play_children['New & Updated'] = []
```

Now we can start extracting data. This section has a certain number of items that should also need to iterate in a loop using the `.TAQqTe` selector:

```python
for item in selector.css('.TAQqTe'):
    # data extraction will be here
```

The difference in data extraction in this function is that data such as `category`, `screenshot_image` and `video` are additionally extracted here. Data is also retrieved by other selectors:

```python
title = item.css('.OnEJge::text').get()
link = 'https://play.google.com' + item.css('::attr(href)').get()
category = item.css('.ubGTjb > .w2kbF::text').getall()[-1]
rating = item.css('.ubGTjb div .w2kbF::text').get()
rating = float(rating) if rating else rating
price = item.css('.ePXqnb::text').get()
price = float(price[1:]) if price else 'free'
thumbnail = item.css('.j2FCNc img::attr(srcset)').get().replace(' 2x', '')
screenshot_image = item.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()
video = item.css('.XdjT2b::attr(data-video-url)').get()
```

After extracting the data, they are appended to the `google_play_children` dictionary by the `'New & Updated'` key:

```python
google_play_children['New & Updated'].append({
    'title': title,
    'link': link,
    'category': category,
    'rating': rating,
    'price': price,
    'thumbnail': thumbnail,
    'screenshot_image': screenshot_image,
    'video': video
})
```

The complete function to scrape New & Updated section would look like this:

```python
def scrape_new_and_updated(selector):
    google_play_children['New & Updated'] = []

    for item in selector.css('.TAQqTe'):
        title = item.css('.OnEJge::text').get()
        link = 'https://play.google.com' + item.css('::attr(href)').get()
        category = item.css('.ubGTjb > .w2kbF::text').getall()[-1]
        rating = item.css('.ubGTjb div .w2kbF::text').get()
        rating = float(rating) if rating else rating
        price = item.css('.ePXqnb::text').get()
        price = float(price[1:]) if price else 'free'
        thumbnail = item.css('.j2FCNc img::attr(srcset)').get().replace(' 2x', '')
        screenshot_image = item.css('.Vc0mnc img::attr(src), .jpDEN::attr(src)').get()
        video = item.css('.XdjT2b::attr(data-video-url)').get()
        
        google_play_children['New & Updated'].append({
            'title': title,
            'link': link,
            'category': category,
            'rating': rating,
            'price': price,
            'thumbnail': thumbnail,
            'screenshot_image': screenshot_image,
            'video': video
        })
```


<h3 id="output">Output</h3>

Output for `scrape_google_play_children()` function:

```json
{
  "New & Updated": [
    {
      "title": "PBS KIDS Video",
      "link": "https://play.google.com/store/apps/details?id=org.pbskids.video",
      "category": "Education",
      "rating": 4.4,
      "price": "free",
      "thumbnail": "https://play-lh.googleusercontent.com/EvR3uJ27B4SwWfK3LfwFrt0Sy3byTrmLHVGRndgJuf8rcvKMRyREhaUcXehQUler5w=s128-rw",
      "screenshot_image": "https://play-lh.googleusercontent.com/9MSE2M5sGVy73d75bBemSfZQicBp1cOkjjG-c3tvW5vOVrpOaXdAyjmnbVcBCMWSaLk=w416-h235-rw",
      "video": null
    },
    ... other items
  ],
  "Encourage kindness": [
    {
      "title": "Breathe, Think, Do with Sesame",
      "link": "https://play.google.com/store/apps/details?id=air.com.sesameworkshop.ResilienceThinkBreathDo",
      "rating": 4.0,
      "price": "free",
      "thumbnail": "https://play-lh.googleusercontent.com/-UbCkW4xbM661t4mndTi7owhXY0GYBCRQn4Pxl7_1tXgCCvqKsJwUKE-O61NO0CuJA=s512-rw"
    },
    ... other items
  ],
  ... other sections
  "Fan favorites": [
    {
      "title": "Polar Bear Horizon",
      "link": "https://play.google.com/store/apps/details?id=com.oceanhouse_media.booksmpolarbear_app",
      "rating": 4.7,
      "price": 2.99,
      "thumbnail": "https://play-lh.googleusercontent.com/hnupPIokoza2NOF3faGTeNImvFY2Lw_0aeSwi9a-_yG6e0GpixCgOZ_vNEPwwnXHMSw=s512-rw"
    },
    ... other items
  ]
}
```

<h2 id="google_play_children">Using Google Play Apps Store API from SerpApi</h2>

This section is to show the comparison between the DIY solution and our solution.

The main difference is that it's a quicker approach. [Google Play Apps Store API](https://serpapi.com/google-play-apps) will bypass blocks from search engines and you don't have to create the parser from scratch and maintain it.

First, we need to install [`google-search-results`](https://pypi.org/project/google-search-results/):

```lang-none
pip install google-search-results
```

Import the necessary libraries for work:

```python
from serpapi import GoogleSearch
import os, json
```

Next, we write a search query and the necessary parameters for making a request:

```python
params = {
    # https://docs.python.org/3/library/os.html#os.getenv
    'api_key': os.getenv('API_KEY'),    # your serpapi api
    'engine': 'google_play',			# SerpApi search engine
    'store': 'apps',					# Google Play store
    'apps_category': 'FAMILY'			# parameter defines the apps and games store category. In this case we use 'FAMILY' to scrape Google Play Children apps
}
```

The GIF below shows where to select this category in the [playground](https://serpapi.com/playground?engine=google_play&gl=us&hl=en&store=apps&apps_category=FAMILY):

![show-how-select-category](https://user-images.githubusercontent.com/78694043/195514020-1ed77151-d145-46f7-81e6-39acce4029a3.gif)

We then create a `search` object where the data is retrieved from the SerpApi backend. In the `result_dict` dictionary we get data from JSON:

```python
search = GoogleSearch(params)
result_dict = search.get_dict()
```

The data is retrieved quite simply, we just need to turn to the corresponding key. All sections with required data are in the `'organic_results'` key, so you need to iterate over them. For each section, we extract the `section_title` and make it a key in the `google_play_children` dictionary, where a list of extracted data for each item will be added later:

```python
google_play_children = {}
	
for result in result_dict['organic_results']:
    section_title = result['title']
    google_play_children[section_title] = []
```

The `'items'` key contains data about each item in this section. Therefore, it also needs to be iterated in a loop. To get the data, you need to refer to the corresponding key. 

Sometimes, data such as `rating`, `price`, `extracted_price`, `category` or `video` is missing. 

Therefore, the [`dict.get()`](https://www.w3schools.com/python/ref_dictionary_get.asp) method was used to get it, which by default returns `None` if there is no data. This will look much better than [exception handling](https://docs.python.org/3/tutorial/errors.html#handling-exceptions) which is also used to prevent errors but makes the code less readable and more cumbersome:

```python   
for item in result['items']:
    google_play_children[section_title].append({
        'title': item['title'],
        'link': item['link'],
        'product_id': item['product_id'],
        'serpapi_link': item['serpapi_link'],
        'rating': item.get('rating'),
        'price': item.get('price', 'free'),
        'extracted_price': item.get('extracted_price', 0),
        'category': item.get('category'),
        'video': item.get('video'),
        'thumbnail': item['title'],
    })
```

The `dict.get(keyname, value)` method can be used to retrieve all data from the dictionary, but I decided to demonstrate possible errors in getting data that a user might encounter.

Example code to integrate:

```python
from serpapi import GoogleSearch
import os, json

params = {
    # https://docs.python.org/3/library/os.html#os.getenv
    'api_key': os.getenv('API_KEY'),    # your serpapi api
    'engine': 'google_play',			# SerpApi search engine
    'store': 'apps',					# Google Play store
    'apps_category': 'FAMILY'			# parameter defines the apps and games store category. In this case we use 'FAMILY' to scrape Google Play Children apps
}

search = GoogleSearch(params)           # where data extraction happens on the SerpApi backend
result_dict = search.get_dict()         # JSON -> Python dict

google_play_children = {}

for result in result_dict['organic_results']:
    section_title = result['title']
    google_play_children[section_title] = []
    
    for item in result['items']:
        google_play_children[section_title].append({
            'title': item['title'],
            'link': item['link'],
            'product_id': item['product_id'],
            'serpapi_link': item['serpapi_link'],
            'rating': item.get('rating'),
            'price': item.get('price', 'free'),
            'extracted_price': item.get('extracted_price', 0),
            'category': item.get('category'),
            'video': item.get('video'),
            'thumbnail': item['title'],
        })

print(json.dumps(google_play_children, indent=2, ensure_ascii=False))
```

Output:

```json
{
  "New & updated": [
    {
      "title": "PBS KIDS Video",
      "link": "https://play.google.com/store/apps/details?id=org.pbskids.video",
      "product_id": "org.pbskids.video",
      "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=org.pbskids.video&store=apps",
      "rating": 4.4,
      "price": "free",
      "extracted_price": 0,
      "category": "Education",
      "video": null,
      "thumbnail": "PBS KIDS Video"
    },
    ... other items
    {
      "title": "Little Panda's Toy Adventure",
      "link": "https://play.google.com/store/apps/details?id=com.sinyee.babybus.spliceII",
      "product_id": "com.sinyee.babybus.spliceII",
      "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=com.sinyee.babybus.spliceII&store=apps",
      "rating": 3.3,
      "price": "free",
      "extracted_price": 0,
      "category": "Educational",
      "video": "https://play.google.com/video/lava/web/player/yt:movie:hEEnvV2nOCI?autoplay=1&embed=play",
      "thumbnail": "Little Panda's Toy Adventure"
    }
  ],
  ... other sections
  "Recommended for you": [
    {
      "title": "Toca Kitchen 2",
      "link": "https://play.google.com/store/apps/details?id=com.tocaboca.tocakitchen2",
      "product_id": "com.tocaboca.tocakitchen2",
      "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=com.tocaboca.tocakitchen2&store=apps",
      "rating": 4.1,
      "price": "free",
      "extracted_price": 0,
      "category": null,
      "video": null,
      "thumbnail": "Toca Kitchen 2"
    },
    ... other items
    {
      "title": "Math Kids: Math Games For Kids",
      "link": "https://play.google.com/store/apps/details?id=com.rvappstudios.math.kids.counting",
      "product_id": "com.rvappstudios.math.kids.counting",
      "serpapi_link": "https://serpapi.com/search.json?engine=google_play_product&gl=us&hl=en&product_id=com.rvappstudios.math.kids.counting&store=apps",
      "rating": 4.0,
      "price": "free",
      "extracted_price": 0,
      "category": null,
      "video": null,
      "thumbnail": "Math Kids: Math Games For Kids"
    }
  ]
}
```

<h2 id="links">Links</h2>

- [Code in the online IDE](https://replit.com/@chukhraiartur/blog-google-play-children#main.py)
- [Google Play Apps Store API](https://serpapi.com/google-play-apps)


<p style="text-align: center;">Join us on <a href="https://twitter.com/serp_api">Twitter</a> | <a href="https://www.youtube.com/channel/UCUgIHlYBOD3yA3yDIRhg_mg">YouTube</a></p>

<p style="text-align: center;">Add a  <a href="https://github.com/serpapi/public-roadmap/issues">Feature Request</a>💫 or a <a href="https://github.com/serpapi/public-roadmap/issues">Bug</a>🐞</p>