Scoring

class ScoreTypes(models.TextChoices):
    RELATIVE_LEGACY = "relative_legacy"
    PEER = "peer"
    BASELINE = "baseline"
    SPOT_PEER = "spot_peer"
    SPOT_BASELINE = "spot_baseline"
    MANUAL = "manual"

def evaluate_forecasts_peer_accuracy(
    forecasts: Sequence[Forecast | AggregateForecast],
    base_forecasts: list[Forecast | AggregateForecast] | None,
    resolution_bucket: int,
    forecast_horizon_start: float,
    actual_close_time: float,
    forecast_horizon_end: float,
    question_type: str,
    geometric_means: list[AggregationEntry] | None = None,
) -> list[ForecastScore]:
    # Calculate geometric mean of all forecasters
    geometric_mean_forecasts = geometric_means or get_geometric_means(base_forecasts)
    
    # Score each forecast against the community
    for forecast in forecasts:
        forecast_start = max(forecast.start_time.timestamp(), forecast_horizon_start)
        forecast_end = (
            actual_close_time if forecast.end_time is None
            else min(forecast.end_time.timestamp(), actual_close_time)
        )
        
        pmf = forecast.get_pmf()
        p = pmf[resolution_bucket]  # Your forecast probability
        
        # Score against community at each timestep
        for gm in geometric_mean_forecasts:
            if forecast_start <= gm.timestamp < forecast_end:
                gmp = gm.pmf[resolution_bucket]  # Community forecast probability
                
                # Peer score formula
                interval_score = (
                    100 * (gm.num_forecasters / (gm.num_forecasters - 1)) 
                    * np.log(p / gmp)
                )
                
                if question_type in QUESTION_CONTINUOUS_TYPES:
                    interval_score /= 2  # Continuous questions are harder

forecast_duration = forecast_end - forecast_start
forecast_coverage = forecast_duration / total_duration
final_score = interval_score * forecast_coverage

def evaluate_forecasts_baseline_accuracy(
    forecasts: Sequence[Forecast | AggregateForecast],
    resolution_bucket: int,
    forecast_horizon_start: float,
    actual_close_time: float,
    forecast_horizon_end: float,
    question_type: str,
    open_bounds_count: int,
) -> list[ForecastScore]:
    total_duration = forecast_horizon_end - forecast_horizon_start
    
    for forecast in forecasts:
        forecast_coverage = forecast_duration / total_duration
        pmf = forecast.get_pmf()
        
        if question_type in ["binary", "multiple_choice"]:
            # Baseline: uniform distribution over options
            options_at_time = sum(~np.isnan(pmf))
            p = pmf[resolution_bucket]
            
            # Logarithmic score vs. uniform
            forecast_score = (
                100 * np.log(p * options_at_time) / np.log(options_at_time)
            )
        else:
            # Baseline: 5% probability in each tail, uniform over middle
            if resolution_bucket in [0, len(pmf) - 1]:
                baseline = 0.05
            else:
                baseline = (1 - 0.05 * open_bounds_count) / (len(pmf) - 2)
            
            # Logarithmic score vs. baseline
            forecast_score = 100 * np.log(pmf[resolution_bucket] / baseline) / 2
        
        # Weight by coverage
        final_score = forecast_score * forecast_coverage

def get_spot_scoring_time(self) -> datetime | None:
    if self.spot_scoring_time:
        return self.spot_scoring_time
    elif self.cp_reveal_time and self.open_time and self.cp_reveal_time > self.open_time:
        return self.cp_reveal_time
    elif self.actual_close_time:
        return self.actual_close_time
    elif self.scheduled_close_time:
        return self.scheduled_close_time
    return None

def evaluate_forecasts_baseline_spot_forecast(
    forecasts: Sequence[Forecast | AggregateForecast],
    resolution_bucket: int,
    spot_forecast_timestamp: float,
    question_type: str,
    open_bounds_count: int,
) -> list[ForecastScore]:
    for forecast in forecasts:
        start = forecast.start_time.timestamp()
        end = float("inf") if forecast.end_time is None else forecast.end_time.timestamp()
        
        # Only score forecasts active at the spot time
        if start <= spot_forecast_timestamp < end:
            pmf = forecast.get_pmf()
            # Same scoring formula as baseline, but coverage = 1.0
            forecast_scores.append(ForecastScore(forecast_score, 1.0))
        else:
            forecast_scores.append(ForecastScore(0))

class Score(TimeStampedModel):
    user = models.ForeignKey(User, null=True, on_delete=models.CASCADE)
    question = models.ForeignKey(Question, on_delete=models.CASCADE)
    
    # Score data
    score = models.FloatField()
    coverage = models.FloatField(default=0)  # Fraction of forecast period covered
    
    # Metadata
    score_type = models.CharField(max_length=200, choices=ScoreTypes.choices)
    aggregation_method = models.CharField(
        max_length=200,
        choices=AggregationMethod.choices,
        db_index=True
    )

class ArchivedScore(TimeStampedModel):
    user = models.ForeignKey(User, null=True, on_delete=models.CASCADE)
    question = models.ForeignKey(Question, on_delete=models.CASCADE)
    score = models.FloatField()
    coverage = models.FloatField(default=0)
    score_type = models.CharField(max_length=200, choices=ArchivedScoreTypes.choices)
    aggregation_method = models.CharField(max_length=200, choices=AggregationMethod.choices)

class ArchivedScoreTypes(models.TextChoices):
    RELATIVE_LEGACY = "relative_legacy"  # Old scoring system

default_score_type = models.CharField(
    max_length=20,
    choices=ScoreTypes.choices,
    default=ScoreTypes.PEER,
    help_text="""Default score type for this question.
    Generally, this should be either 'peer' or 'spot_peer'.
    Determines which score will be most prominently displayed in the UI.
    Also, for Leaderboards that have a 'score type' of 'default',
    this question's default score type will be the one that contributes
    to the leaderboard.
    """,
)

question_weight = models.FloatField(default=1.0)

class UnsuccessfulResolutionType(models.TextChoices):
    AMBIGUOUS = "ambiguous"
    ANNULLED = "annulled"