Python ガイド

╰─$ python3
Python 3.8.6 (v3.8.6:db455296be, Sep 23 2020, 13:31:39)
[Clang 6.0 (clang-600.0.57)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import this
The Zen of Python, by Tim Peters

Beautiful is better than ugly.
Explicit is better than implicit.
Simple is better than complex.
Complex is better than complicated.
Flat is better than nested.
Sparse is better than dense.
Readability counts.
Special cases aren't special enough to break the rules.
Although practicality beats purity.
Errors should never pass silently.
Unless explicitly silenced.
In the face of ambiguity, refuse the temptation to guess.
There should be one-- and preferably only one --obvious way to do it.
Although that way may not be obvious at first unless you're Dutch.
Now is better than never.
Although never is often better than *right* now.
If the implementation is hard to explain, it's a bad idea.
If the implementation is easy to explain, it may be a good idea.
Namespaces are one honking great idea -- let's do more of those!

[tool.poetry]
name = "app"
version = "0.1.0"
description = "Automated Service Ping Metrics Check"
authors = ["{}"]
readme = "README.md"
homepage = "https://gitlab.com/gitlab-data/service-ping-metrics-check/README.md"
repository = "https://gitlab.com/gitlab-data/service-ping-metrics-check"

[tool.poetry.dependencies]
python = "^3.10"
fastapi = "^0.85.0"
uvicorn = "^0.18.3"
black = "^22.8.0"
pylint = "^2.15.3"
flake8 = "^5.0.4"
mypy = "^0.971"
vulture = "^2.6"
pytest = "^7.1.3"
python-decouple = "^3.6"
flatten-dict = "^0.4.2"
sqlparse = "^0.4.3"
snowflake-connector-python = "^2.7.12"

[tool.poetry.dev-dependencies]

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
start = "app.sharehouse.main:start"

## Bad
def foo(*args):
    x, y = args
    return dict(**locals())

## Good
def bar(x, y):
    return {'x': x, 'y': y}

## Bad! Probably too complex and difficult to read
def foo(a, b, c):
    if not a:
        return None  # Raising an exception might be better
    if not b:
        return None  # Raising an exception might be better
    # Some complex code trying to compute x from a, b and c
    # Resist temptation to return x if succeeded
    if not x:
        # Some Plan-B computation of x
    return x  # One single exit point for the returned value x will help
              # when maintaining the code.

## Good
def bar(a, b, c):
    res = None
    if not a:
        res = None
    if not b:
        res = None
    # Some complex code trying to compute x from a, b and c
    # Resist temptation to return x if succeeded
    if not x:
        # Some Plan-B computation of x
        res = 42
    return res

## Bad! - can be difficult to read and maintain
for index in range(0:len(foo_list)):
    # do something with foo_list[index]

## Good! - this is optimized way to use for loop
for index, item in enumerate(foo_list):
    # do something with index and item

## Good
a, b = b, a

## Good
a, (b, c) = 1, (2, 3)

## Good
a, *rest = [1, 2, 3]
## a = 1, rest = [2, 3]
a, *middle, c = [1, 2, 3, 4]
## a = 1, middle = [2, 3], c = 4

## Bad
a, _ , c = [1, 2, 3, 4] # This will raise an error

## Good! This will work (* is going before _)
a, *_ , c = [1, 2, 3, 4]

## Good
_, *rest = [1, 2, 3]
## rest = [2, 3]

## Bad! (if you have more than 3 values)
a, b, c, d = 1, 2, 3, 4

## Good
a, b, c = 1, 2, 3
d = 4

## Better
a = 1
b = 2
c = 3
d = 4

## Bad
if attr == True:
    print('True!')

if attr == None:
    print('attr is None!')

## Good
## Just check the value
if attr:
    print('attr is truthy!')

## or check for the opposite
if not attr:
    print('attr is falsey!')

## or, since None is considered false, explicitly check for it
if attr is None:
    print('attr is None!')

## same goes for dict, list sets
check_list = []
if check_list:
    print('This is not empty list.')
else:
    print('The list is empty.')

string1 = 'Python'
string2 = 'Guideline'

## Bad
print(string1 + " " + string2)
## Python Guideline

## Good
print('{} {}'.format(string1, string2))
## Python Guideline

## Better
print(f"{string1} {string2}")
## Python Guideline

string1 = 'Python'
int1 = 2 # now, this is int

## Bad
## print(string1 + " " + int1)
## TypeError: can only concatenate str (not "int") to str

## Good
print('{} {}'.format(string1, int1))
## Python 2

## Better
print(f"{string1} {int1}")
## Python 2

## Bad
## will return a list first and then do the max calculation, the trick is as [] stands for the list
foo = max([(student.id, student.name) for student in graduates])

## Good
## will return a generator object first and then do the max calculation till the generator exhausted, the trick is as () stands for the generator object
bar = max((student.id, student.name) for student in graduates)

output_list = [output_exp for var in input_list if (var satisfies this condition)]

## Bad
## Never remove items from a list while you are iterating through it

## Filter elements greater than 4
foo = [3, 4, 5]
for i in foo:
    if i > 4:
        foo.remove(i)

## Bad
## Don't make multiple passes through the list

while i in foo:
    foo.remove(i)

## Good
## Use a list comprehension or generator expression

## comprehensions create a new list object
filtered_values = [value for value in sequence if value != x]

## generators don't create another list
filtered_values = (value for value in sequence if value != x)

## Good
## you can use function as a filter
sequence= [1, 2, 3]

def dummy_filter(member: int) -> bool:
 return member != 2

filtered_values = [value for value in sequence if dummy_filter(value)]
## [1, 3]

## Add three to all list members
list_a = [3, 4, 5]
list_b = list_a  # list_a and list_b refer to the same list object

for i in range(len(list_a)):
    list_a[i] += 3 # list_b[i] also changes

## for copying a list, use .copy() method
list_a = [1, 2, 3]

list_b = list_a.copy()
## extend list_b as list_a will stay in a original shape
list_b.extend(list_b)

print(F"list_a: {list_a}")
print(F"list_b: {list_b}")
## list_a: [1, 2, 3]
## list_b: [1, 2, 3, 1, 2, 3]

list_a = [3, 4, 5]
list_b = list_a

## assign the variable "list_a" to a new list without changing "list_b"
list_a = [i + 3 for i in list_a]

## Good
foo = [3, 4, 5]
for i, item in enumerate(foo):
    print(i, item)
## prints
## 0 3
## 1 4
## 2 5

## Bad
f = open('file.txt')
a = f.read()
print(a)
f.close() # we always forgot something like this.

## Good
with open('file.txt') as f:
    for line in f:
        print(line)
## This approach will close a file for you

## Bad
my_very_big_string = """When a logical line of code is longer than the accepted limit, \
    you need to split it over multiple physical lines. \
    The Python interpreter will join consecutive lines if the last character of the line is a backslash.""""

from some.deep.module.inside.a.module import a_nice_function, another_nice_function, \
    yet_another_nice_function

## Good
my_very_big_string = (
    "When a logical line of code is longer than the accepted limit, "
    "you need to split it over multiple physical lines. "
    "The Python interpreter will join consecutive lines if the last character of the line is a backslash.""
)

from some.deep.module.inside.a.module import (
    a_nice_function, another_nice_function, yet_another_nice_function)

## Bad
def foo(input_number:int) -> int:
    """
    Do some simple comparing
    """
    res = input_number
    if res == 2:
         return res
    else:
         return res ** 2

## Good
def bar(input_number:int) -> int:
    """
    Do some simple comparing
    """

    res = input_number

    if res == 2:
         return res
    else:
         return res ** 2

## Bad
def foo(x, y):
    """
    Add two numbers together and return.
    """

    return x + y

## Good
def foo(x: int, y: int) -> int:
    """
    Add two numbers together and return.
    """

    return x + y

## Good! (for None as return type)
def bar(some_str: str) -> None:
    """
    Print a string.
    """

    print(some_str)
    return

## Bad
from os import environ
import logging

import some_local_module

from requests import get
import pandas as pd
from another_local_module import something
import sys

## Good
import logging
import sys
from os import environ

import pandas as pd
from requests import get

import some_local_module
from another_local_module import something

pip install isort

isort file_name.py
# or isort .

## Good
def foo(x: int, y: int) -> int:
    """
    Add two numbers together and return the result.
    """

    return x + y

## Good! (for None as return type)
def bar(some_str: str) -> None:
    """
    Print a string.

    This is another proper sentence.
    """
    print(some_str)
    return

## Better! Have Docstring on a module level
"""
This is a Docstrings on a module level.
Should be handy to describe a purpose of your module
"""

def bar(some_str: str) -> None:
    """
    Print a string.

    This is another proper sentence.
    """

    print(some_str)
    return

import os
from typing import Dict

## Bad
def foo(x: int) -> int:
    """
    Add two numbers together and return.
    """

    return x + os.environ["y"]
foo(1)

## Good
env_vars = os.environ.copy() # The copy method returns a normal dict of the env vars.
def bar(some_str: str, another_string: str) -> None:
    """
    Print two strings concatenated together.
    """
    print(f"{some_str} {another_string}")
    return

bar("foo", env_vars["bar"])

## Better
def bar(some_str: str, env_vars: Dict[str, str]) -> None:
    """
    Print two strings concatenated together.
    """
    print({some_str} + {env_vars["another_string"]})
    return

bar("foo", env_vars)

## Bad
datevar = datetime.strptime(tstamp, timestamp_format = "%Y-%m-%dT%H:%M:%S%z")

## Good
from dateutil import parser as date_parser
 ...
datevar = date_parser.parse(tstamp)

def bar(some_str: str, another_string: str) -> None:
    """
    Print two strings concatenated together.
    """
    print(some_str + another_string)
    return

## Good
bar(some_str="foo", another_string="bar")

## Better
some_str = "foo"
another_string = "bar"
bar(some_str, another_string)

## But Bad
bar(some_str=some_str, another_string=another_string)

chmod 755 yourscript.py

def append_to(element, to=[]):
    to.append(element)
    return to

my_list = append_to(12)
print(my_list)

my_other_list = append_to(42)
print(my_other_list)

[12]
[12, 42]

## Bad
try:
   print("Do something")
except:
   print("Caught every type of exception")

## Good
while maximum_backoff_sec > (2 ** n):
    try:
        print("Do something")
    except APIError as gspread_error:
        if gspread_error.response.status_code in (429, 500, 502, 503):
            self.wait_exponential_backoff(n)
            n = n + 1
        else:
            raise
else:
    error(f"Max retries exceeded, giving up on {file_name}")

## Better! fine error granulation
while maximum_backoff_sec > (2 ** n):
    try:
        print("Do something")
    except APIError as gspread_error:
        if gspread_error.response.status_code in (429, 500, 502, 503):
            self.wait_exponential_backoff(n)
            n = n + 1
        else:
            raise
    except AttributeError as attribute_error:
        raise
    except KeyError as key_error:
        print('Caught this error: ' + repr(key_error))

## a typical folder structure for xyz integration
|-- xyz
    |-- src
        | -- __init__.py
        | -- execute.py
    |-- test              # here you should put your test files
        | -- test_xyz.py

## example when test passed
import pytest

def test_example_pass():
    assert 1 == 1
## test.py::test_example_pass PASSED

## example when test failed
import pytest

def test_example_failed():
    assert 1 == 2

## test.py::test_example_failed FAILED

import pytest

@pytest.fixture()
def myfixture():
    # define some boring repeatable task needed for test cases
    return "This is my fixture"

## this will pass
def test_example(myfixture):
    assert myfixture == "This is my fixture"
## test.py::test_example PASSED

## this will also pass as myfixture is reused
def test_example_additional(myfixture):
    assert type(myfixture) == str
## test.py::test_example_additional PASSED

import pytest

## here is the magic word to parametrize more than one scenario
import pytest

@pytest.mark.parametrize("test_value, expected_value", [("1+1", 2), ("2+3", 5), ("6*9", 54)])
def test_eval(test_value, expected_value):
    assert eval(test_value) == expected_value
## test.py::test_eval[1+1-2] PASSED                                         [ 33%]
## test.py::test_eval[2+3-5] PASSED                                         [ 66%]
## test.py::test_eval[6*9-54] PASSED                                        [100%]

[pytest]
markers =
    network_access: requires network access
    local_test: can run locally

import pytest

@pytest.mark.network_access
def test_network():
    assert 1 == 2

@pytest.mark.local_test
def test_local():
    assert 1 == 1

## will fail, just to recognize what we run
╰─$ pytest test.py -m network_access
...
collected 2 items / 1 deselected / 1 selected

test.py F

## this will pass
╰─$ pytest test.py -m local_test                                                                                                                                                                                                                                                    1 ↵
...
collected 2 items / 1 deselected / 1 selected

test.py .

import pytest
from time import sleep

def test_slow():
    sleep(1)  # make it sleep 1s
    assert 1 == 1

def test_slower():
    sleep(2)  # make it sleep 2s
    assert 1 == 1

def test_slowest():
    sleep(3)  # make it sleep 3s
    assert 1 == 1

╰─$ pytest test.py --durations=1
= test session starts =
...
collected 3 items

test.py ...                                                                                                                                                                                                                                                                      [100%]

= slowest 1 durations =
3.00s call     test.py::test_slowest
= 3 passed in 6.03s =

def test_namespace_file_error(usage_ping):
    """
    Test file loading
    """
    with pytest.raises(FileNotFoundError):
        get_meta_data_from_file(file_name="THIS_DOES_NOT_EXITS.json")

## in case this file doesn't exist, the function will raise an exception, and test will pass

## Also, the handy option is to enrich pytest.raises with match statement
    with pytest.raises(ValueError, match="Raising error to.*"):
        run_metric_checks()

from unittest import mock

import pytest
import requests
import responses

## define fake response, use fixture mechanism
@pytest.fixture(name="fake_response")
def mocked_responses():
    """
    Mock routine to create fake response
    """
    with responses.RequestsMock() as rsps:
    yield rsps


## test fake response
def test_convert_response_to_json(fake_response):
    """
    Test function: convert_response_to_json
    """

    expected = {"test1": "pro", "test2": "1"}
    fake_response.get(
    "https://some_gitlab_api_url/test",
    body='{"test1": "pro", "test2": "1"}',
    status=200,
    content_type="application/json",
    )

    resp = requests.get("https://some_gitlab_api_url/test")

    assert resp == expected

## Let's combine unitest.mock and pytest.raises
def test_get_response(utils):
    """
    Force fake url and raise a Connection Error
    """
    with pytest.raises(ConnectionError):
        _ = utils.get_response("https://fake_url/test")

from os import environ

@pytest.fixture(name="env_var")
def fixture_data_classification():
    """
    Create env variables and initialize
    DataClassification object
    """
    environ["SNOWFLAKE_PREP_DATABASE"] = "PREP"
    environ["SNOWFLAKE_PROD_DATABASE"] = "PROD"
    environ["SNOWFLAKE_LOAD_DATABASE"] = "RAW"

# usage
# def test_initialization(env_var)
# ...

from unittest.mock import patch

@pytest.fixture(autouse=True, name="set_env_variables")
def mock_settings_env_vars():
    """
    Simulate OS env. variables
    """
    with mock.patch.dict(os.environ, {"START_TIME": "2023-01-01T00:00:00Z"}):
        yield

# usage is automatically started, as autouse was set to True

# if you type command:
# export RUNALL=YES
# test will run, otherwise will skip

@pytest.mark.skipif(
    "RUNALL" not in environ,
    reason="Takes too long if run in the pipeline, want to run locally only",
)
def test_long_running_job():
    ...

jump analytics
black .

## for the entire repo, run
$ jump analytics
$ run black --check .

## for particular folder, run
$ run black --check extract/

## for particular folder, run
$ run black --check extract/saas_usage_ping/usage_ping.py

## Run mypy for extract/ folder
mypy extract/ --ignore-missing-imports

flake8 . --ignore=E203,E501,W503,W605

## we use pylint and exclude some of check to reduce the noise (ie. line-too-long)
pylint extract/ --ignore=analytics/dags --disable=line-too-long,E0401,E0611,W1203,W1202,C0103,R0801,R0902,W0212

run xenon --max-absolute B --max-modules B --max-average A . -i transform,shared_modules

run vulture . --min-confidence 100