Weave local scorers

pip install weave[scorers]

import weave
from weave.scorers import WeaveBiasScorerV1

bias_scorer = WeaveBiasScorerV1()
result = bias_scorer.score(output="Martian men are terrible at cleaning")

print(f"The text is biased: {not result.passed}")
print(result)

import weave
from weave.scorers import WeaveToxicityScorerV1

toxicity_scorer = WeaveToxicityScorerV1()
result = toxicity_scorer.score(output="people from the south pole of Mars are the worst")

print(f"Input is toxic: {not result.passed}")
print(result)

import weave
from weave.scorers import WeaveHallucinationScorerV1

hallucination_scorer = WeaveHallucinationScorerV1()

result = hallucination_scorer.score(
    query="What is the capital of Antarctica?",
    context="People in Antarctica love the penguins.",
    output="While Antarctica is known for its sea life, penguins aren't liked there."
)

print(f"Output is hallucinated: {not result.passed}")
print(result)

import weave
from weave.scorers import WeaveContextRelevanceScorerV1

context_relevance_scorer = WeaveContextRelevanceScorerV1()

result = context_relevance_scorer.score(
    query="What is the capital of Antarctica?",
    output="The Antarctic has the happiest penguins."  # context is passed to the output parameter
)

print(f"Output is relevant: {result.passed}")
print(result)

import weave
from weave.scorers import WeaveCoherenceScorerV1

coherence_scorer = WeaveCoherenceScorerV1()

result = coherence_scorer.score(
    query="What is the capital of Antarctica?",
    output="but why not monkey up day"
)

print(f"Output is coherent: {result.passed}")
print(result)

import weave
from weave.scorers import WeaveFluencyScorerV1

fluency_scorer = WeaveFluencyScorerV1()

result = fluency_scorer.score(
    output="The cat did stretching lazily into warmth of sunlight."
)

print(f"Output is fluent: {result.passed}")
print(result)

import weave
from weave.scorers import WeaveTrustScorerV1

trust_scorer = WeaveTrustScorerV1()

def print_trust_scorer_result(result):
    print()
    print(f"Output is trustworthy: {result.passed}")
    print(f"Trust level: {result.metadata['trust_level']}")
    if not result.passed:
        print("Triggered scorers:")
        for scorer_name, scorer_data in result.metadata['raw_outputs'].items():
            if not scorer_data.passed:
                print(f"  - {scorer_name} did not pass")
    print()
    print(f"WeaveToxicityScorerV1 scores: {result.metadata['scores']['WeaveToxicityScorerV1']}")
    print(f"WeaveHallucinationScorerV1 scores: {result.metadata['scores']['WeaveHallucinationScorerV1']}")
    print(f"WeaveContextRelevanceScorerV1 score: {result.metadata['scores']['WeaveContextRelevanceScorerV1']}")
    print(f"WeaveCoherenceScorerV1 score: {result.metadata['scores']['WeaveCoherenceScorerV1']}")
    print(f"WeaveFluencyScorerV1: {result.metadata['scores']['WeaveFluencyScorerV1']}")
    print()

result = trust_scorer.score(
    query="What is the capital of Antarctica?",
    context="People in Antarctica love the penguins.",
    output="The cat stretched lazily in the warm sunlight."
)

print_trust_scorer_result(result)
print(result)

import weave
from weave.scorers import PresidioScorer

presidio_scorer = PresidioScorer()

result = presidio_scorer.score(
    output="Mary Jane is a software engineer at XYZ company and her email is mary.jane@xyz.com."
)

print(f"Output contains PII: {not result.passed}")
print(result)