Historical Software Peer Review -- pyOpenSci

# Sometimes the token doesn't get updated properly in jupyter. 
# If this happens you can do a few things
# import os
# from dotenv import load_dotenv

# load_dotenv()

# print(os.getenv("GITHUB_TOKEN"))

# # Print all envt keys to see what token value is stored
# for key, value in os. environ.items():
#     print(f"{key}: {value}")

#in bash use: export GITHUB_TOKEN=valuehere

"""
This notebook collects data from the GitHub api and processes issue data. 
I suspect similar to the issue/pr notebook, that it would be better to grab 
all of this data in advance and then process it in the notebook here rather than 
always hitting the API here in this notebook. I'm running into API auth issue 
as myst is always rebuilding the notebook. 
"""
import warnings
from datetime import datetime
from pathlib import Path

import altair as alt
import pandas as pd

from pyosmeta import ProcessIssues
from pyosmeta.github_api import GitHubAPI
from pyosmetrics.plot_theme import load_poppins_font, register_and_enable_poppins_theme

pd.options.mode.chained_assignment = None
pd.options.future.infer_string = True

# Suppress all warnings
warnings.filterwarnings("ignore")

# Load & register Poppins font and theme for the page
load_poppins_font()
register_and_enable_poppins_theme()

label_map = {
    "pre-review": ["New Submission!", "0/pre-review-checks"],
    "seeking editor": ["0/seeking-editor"],
    "under-review": [
        "1/editor-assigned",
        "2/seeking-reviewers",
        "3/reviewers-assigned",
        "4/reviews-in-awaiting-changes",
        "5/awaiting-reviewer-response",
    ],
    "pyos-accepted": ["6/pyos-approved"],
    "joss-accepted": ["9/joss-approved"],
}

current_date = datetime.today().date()
today = current_date.strftime("%d %B %Y")  # Format: YYYY-MM-DD

current_dir = Path.cwd()
parent_dir = current_dir.parent.parent
data_dir = parent_dir / "_data"
file_path = data_dir / "review_submissions.csv"
reviews = pd.read_csv(file_path, parse_dates=["date_opened", "date_closed"])
total_submissions = len(reviews)
reviews.dtypes

file_path = data_dir / "review_presubmissions.csv"
presubmissions = pd.read_csv(file_path)
total_presubmissions = len(presubmissions)

# TODO: check presubmission counts!
presub_count = presubmissions[
    presubmissions["date_closed"].isna()
]
presub_open_count = len(presub_count)

review_status_ct = reviews["status"].value_counts().reset_index()
review_status_ct.rename(columns={"labels": "status"}, inplace=True)
review_status_ct

# This is less accurate because it grabs on hold issues
active_open_reviews = reviews[reviews["status"].isin(["seeking editor", 
                                                            "under-review", 
                                                            "pre-review"])]
open_reviews = active_open_reviews[active_open_reviews["date_closed"].isna()]
open_count = len(open_reviews)

# TODO: double check counts
seeking_editor = open_reviews[
    open_reviews["labels"].apply(lambda x: "0/seeking-editor" in x)
]
seeking_editor_count = len(seeking_editor)

data = pd.DataFrame(
    {
        "Category": ["Open Reviews", "Pre-submission Inquiries"],
        "Count": [open_count, presub_open_count],
    }
)

bar_chart = (
    alt.Chart(data)
    .mark_bar(color="purple")
    .encode(
        x=alt.X("Category:N", title="Category", axis=alt.Axis(labelAngle=0)),
        y=alt.Y("Count:Q", title="Count", axis=alt.Axis(tickCount=5)),
        tooltip=["Category", "Count"],
    )
    .properties(
        title=alt.TitleParams(text="pyOpenSci: Current Open Reviews"), width=600
    )
    .interactive()
)

bar_chart.show()

chart = (
    alt.Chart(review_status_ct)
    .mark_bar()
    .encode(
        y=alt.Y(
            "status",
            title="Review Status",
            sort=[
                "pre-review",
                "seeking editor",
                "under-review",
                "pyos-accepted",
                "joss-accepted",
                "on-hold",
                "out-of-scope",
            ],
        ),
        x=alt.X(
            "count",
            axis=alt.Axis(tickCount=5),
            title="Count",
            scale=alt.Scale(domain=[0, 20]),
        ),
        tooltip=[
            alt.Tooltip("status:N", title="Status"),
            alt.Tooltip("count:Q", title="Count"),
        ],
    )
    .properties(title="Count of Packages by Status", width="container")
)

# Display the chart
chart.show()

# This calculates status on currently open reviews
#open_reviews["status"] = open_reviews["labels"].apply(set_review_status)
status_counts = open_reviews["status"].value_counts().reset_index()

chart = (
    alt.Chart(status_counts)
    .mark_bar()
    .encode(
        x=alt.X("status"),
        y="count",
        tooltip=[
            alt.Tooltip("status:N", title="Status"),
            alt.Tooltip("count:Q", title="Count"),
        ],
    )
    .properties(title="Review Status for Current Reviews", width="container")
)

# Display the chart
chart.show()

# Should be 71 reviews and 56 presubmissions?
# Total presubmissions - get the total number of pre-submission inquiries (all time)
# Get presubmissions separately

# Extract year and quarter from the 'date_opened' column
reviews.loc[:, "year_quarter"] = reviews["date_opened"].dt.to_period("Q")
quarterly_counts = reviews["year_quarter"].value_counts().sort_index().reset_index()

quarterly_counts["year_quarter"] = quarterly_counts["year_quarter"].astype(str)

axis_labels = """datum.label == '2019Q1' ? '2019 Q1' : 
                datum.label == '2020Q1' ? '2020 Q1' : 
                datum.label == '2021Q1' ? '2021 Q1' : 
                datum.label == '2022Q1' ? '2022 Q1' :
                datum.label == '2023Q1' ? '2023 Q1' :
                datum.label == '2024Q1' ? '2024 Q1' :
                '' """
chart = (
    alt.Chart(quarterly_counts)
    .mark_bar(color="purple")
    .encode(
        x=alt.X(
            "year_quarter:O",
            title="Year-Quarter",
            axis=alt.Axis(
                labelAngle=0,
                labelExpr=axis_labels,
            ),
        ),
        y=alt.Y(
            "count:Q",
            title="Number of Submissions",
            axis=alt.Axis(tickCount=4, tickMinStep=4),
            scale=alt.Scale(domain=[0, 12]),
        ),
        tooltip=[
            alt.Tooltip("year_quarter:O", title="Quarter"),
            alt.Tooltip("count:Q", title="Number of Issues"),
        ],
    )
    .properties(
        title="Number of Submissions by Quarter per Year",
        width="container",
        height=400,
    )
)

chart.show()

# Group issues by year and get counts
annual_issues = reviews.copy()

# Create a new column 'year' by extracting the year from the 'date_opened' column
annual_issues.loc[:, "year"] = annual_issues["date_opened"].dt.year

# Add year / month
annual_issues["year_month"] = annual_issues["date_opened"].dt.to_period("M")
counts_month_year = annual_issues.groupby("year_month").size().reset_index(name="count")

# Create a complete range of year_month periods
# Note i use this below - don't have to recalculate
all_month_years = pd.period_range(
    start=counts_month_year["year_month"].min(),
    end=counts_month_year["year_month"].max(),
    freq="M",
)

issues_by_year = (
    annual_issues.groupby("year")
    .size()
    .reset_index(name="count")
    .sort_values(by="year", ascending=False)
    .reset_index(drop=True)
)

# Create an Altair bar chart
chart = (
    alt.Chart(issues_by_year)
    .mark_bar(color="purple")
    .encode(
        x=alt.X(
            "year:O",
            axis=alt.Axis(labelAngle=0, labelFontSize=14, titleFontSize=18),
            sort=alt.EncodingSortField(field="year", order="ascending"),
        ),
        y=alt.Y(
            "count:Q",
            axis=alt.Axis(labelFontSize=14, titleFontSize=18, tickCount=5),
        ),
        tooltip=["year", "count"],
    )
    .properties(
        title=alt.TitleParams(
            text="pyOpenSci -- Number of Issues by Year", fontSize=24
        ),
        width=600,
    )
)

chart.show()

# Get fill in months with no issues with a value of 0
month_year_counts = (
    counts_month_year.set_index("year_month")
    .reindex(all_month_years, fill_value=0)
    .rename_axis("year_month")
    .reset_index()
)

# Split year_month into separate year and month columns
month_year_counts["year"] = month_year_counts["year_month"].dt.year
month_year_counts["month"] = month_year_counts["year_month"].dt.strftime("%b")
month_year_counts["month_cat"] = pd.Categorical(
    month_year_counts["month"],
    categories=[
        "Jan",
        "Feb",
        "Mar",
        "Apr",
        "May",
        "Jun",
        "Jul",
        "Aug",
        "Sep",
        "Oct",
        "Nov",
        "Dec",
    ],
    ordered=True,
)
month_year_counts = month_year_counts.drop(columns=["year_month"])

month_year_counts

# Summarize total counts per month
monthly_counts = (
    month_year_counts.groupby("month")["count"]
    .sum()
    .reindex(
        [
            "Jan",
            "Feb",
            "Mar",
            "Apr",
            "May",
            "Jun",
            "Jul",
            "Aug",
            "Sep",
            "Oct",
            "Nov",
            "Dec",
        ],
        fill_value=0,
    )
    .reset_index()
)

# Create the Altair plot
chart = (
    alt.Chart(monthly_counts)
    .mark_bar()
    .encode(
        x=alt.X(
            "month",
            sort=[
                "Jan",
                "Feb",
                "Mar",
                "Apr",
                "May",
                "Jun",
                "Jul",
                "Aug",
                "Sep",
                "Oct",
                "Nov",
                "Dec",
            ],
        ),
        y="count",
        tooltip=["month", "count"],
    )
    .properties(title="Total Counts per Month (2019-2024)", width="container", height=400)
    .configure_axis(labelAngle=0)
)

chart.show()

# Set 'date_opened' column as index / add month and year cols for grouping
monthly_issues = reviews.copy()
monthly_issues["month"] = monthly_issues["date_opened"].dt.month
monthly_issues["year"] = monthly_issues["date_opened"].dt.year
# Get monthly counts
monthly_issues_index = monthly_issues.copy()

monthly_issues_index.set_index(
    monthly_issues_index["date_opened"].dt.to_period("M").dt.strftime("%Y-%m"),
    inplace=True,
)

# Group by the new index (month-year) and count the number of issues for each month-year
monthly_counts = monthly_issues_index.groupby(level=0).size()

# Create a df for every month/year combo in our dataset - this ensures a date for every
# month even if some months are missing
all_month_years = pd.date_range(
    start=monthly_issues.date_opened.min().strftime("%Y-%m"),
    end=monthly_issues.date_opened.max().strftime("%Y-%m"),
    freq="MS",
).to_period("M")

final_monthly = monthly_counts.copy()
# Ensure the index is of type periodIndex to support reindexing
final_monthly.index = pd.PeriodIndex(final_monthly.index, freq="M")
final_monthly = final_monthly.reindex(all_month_years, fill_value=0).to_frame(
    name="issue_count"
)

# Calculate cumulative sum of issue count
final_monthly["cumulative_count"] = final_monthly["issue_count"].cumsum()
final_monthly.reset_index(inplace=True, names="date")
final_monthly["date"] = final_monthly["date"].dt.to_timestamp()

# Create an Altair line plot
chart = (
    alt.Chart(final_monthly)
    .mark_line(color="purple", strokeWidth=8)
    .encode(
        x=alt.X(
            "date:T",
            axis=alt.Axis(
                title="Month",
                format="%b-%Y",
                tickCount="year",
            ),
        ),
        y=alt.Y(
            "cumulative_count:Q",
            axis=alt.Axis(
                title="Number of Issues",
                #tickMinStep=10,  # Not sure why altair isn't recognizing
                tickCount=10,
            ),
        ),
        tooltip=[
            alt.Tooltip("date:T", title="Month"),
            alt.Tooltip("cumulative_count:Q", title="Number of Issues"),
        ],
    )
    .properties(
        title=alt.TitleParams(
            text="Cumulative Review Submissions Over Time",
        ),
        width=600,
        height=400,
    )
)

label = (
    alt.Chart(
        pd.DataFrame(
            {
                "date": [
                    pd.Timestamp("2023-01-01")
                ],  # Specific x-axis location (January 2023)
                "cumulative_count": [16],
                "label": ["Full Time Funding"],
            }
        )
    )
    .mark_text(align="left", dx=5, dy=-10, color="black", fontSize=12, font="Poppins")
    .encode(x="date:T", y="cumulative_count:Q", text="label:N")
)

# Adding an arrow using mark_rule and mark_point
arrow = (
    alt.Chart(
        pd.DataFrame(
            {
                "date": [pd.Timestamp("2023-01-01")],
                "cumulative_count": [
                    20
                ],  # Adjust y-axis location to position the arrow
            }
        )
    )
    .mark_point(shape="triangle", angle=0, size=50, color="black")
    .encode(x="date:T", y="cumulative_count:Q")
)

# Combine the chart and the label
final_chart = chart + label + arrow

# Show the final chart
final_chart.show()

# Show the chart
# chart.show()