243 行
7.8 KiB
TypeScript
243 行
7.8 KiB
TypeScript
export type ActionType =
|
|
| "forehand"
|
|
| "backhand"
|
|
| "serve"
|
|
| "volley"
|
|
| "overhead"
|
|
| "slice"
|
|
| "lob"
|
|
| "unknown";
|
|
|
|
export type Point = {
|
|
x: number;
|
|
y: number;
|
|
visibility?: number;
|
|
};
|
|
|
|
export type TrackingState = {
|
|
prevTimestamp?: number;
|
|
prevRightWrist?: Point;
|
|
prevLeftWrist?: Point;
|
|
prevHipCenter?: Point;
|
|
};
|
|
|
|
export type ActionObservation = {
|
|
action: ActionType;
|
|
confidence: number;
|
|
};
|
|
|
|
export type ActionFrame = {
|
|
action: ActionType;
|
|
confidence: number;
|
|
};
|
|
|
|
export const ACTION_LABELS: Record<ActionType, string> = {
|
|
forehand: "正手挥拍",
|
|
backhand: "反手挥拍",
|
|
serve: "发球",
|
|
volley: "截击",
|
|
overhead: "高压",
|
|
slice: "切削",
|
|
lob: "挑高球",
|
|
unknown: "未知动作",
|
|
};
|
|
|
|
function clamp(value: number, min: number, max: number) {
|
|
return Math.max(min, Math.min(max, value));
|
|
}
|
|
|
|
function distance(a?: Point, b?: Point) {
|
|
if (!a || !b) return 0;
|
|
const dx = a.x - b.x;
|
|
const dy = a.y - b.y;
|
|
return Math.sqrt(dx * dx + dy * dy);
|
|
}
|
|
|
|
function getAngle(a?: Point, b?: Point, c?: Point) {
|
|
if (!a || !b || !c) return 0;
|
|
const radians = Math.atan2(c.y - b.y, c.x - b.x) - Math.atan2(a.y - b.y, a.x - b.x);
|
|
let angle = Math.abs((radians * 180) / Math.PI);
|
|
if (angle > 180) angle = 360 - angle;
|
|
return angle;
|
|
}
|
|
|
|
export function recognizeActionFrame(landmarks: Point[], tracking: TrackingState, timestamp: number): ActionFrame {
|
|
const nose = landmarks[0];
|
|
const leftShoulder = landmarks[11];
|
|
const rightShoulder = landmarks[12];
|
|
const leftElbow = landmarks[13];
|
|
const rightElbow = landmarks[14];
|
|
const leftWrist = landmarks[15];
|
|
const rightWrist = landmarks[16];
|
|
const leftHip = landmarks[23];
|
|
const rightHip = landmarks[24];
|
|
const leftKnee = landmarks[25];
|
|
const rightKnee = landmarks[26];
|
|
const leftAnkle = landmarks[27];
|
|
const rightAnkle = landmarks[28];
|
|
|
|
const hipCenter = {
|
|
x: ((leftHip?.x ?? 0.5) + (rightHip?.x ?? 0.5)) / 2,
|
|
y: ((leftHip?.y ?? 0.7) + (rightHip?.y ?? 0.7)) / 2,
|
|
};
|
|
|
|
const dtMs = tracking.prevTimestamp ? Math.max(16, timestamp - tracking.prevTimestamp) : 33;
|
|
const rightSpeed = distance(rightWrist, tracking.prevRightWrist) * (1000 / dtMs);
|
|
const leftSpeed = distance(leftWrist, tracking.prevLeftWrist) * (1000 / dtMs);
|
|
const hipSpeed = distance(hipCenter, tracking.prevHipCenter) * (1000 / dtMs);
|
|
const rightVerticalMotion = tracking.prevRightWrist ? tracking.prevRightWrist.y - (rightWrist?.y ?? tracking.prevRightWrist.y) : 0;
|
|
|
|
const shoulderTilt = Math.abs((leftShoulder?.y ?? 0.3) - (rightShoulder?.y ?? 0.3));
|
|
const hipTilt = Math.abs((leftHip?.y ?? 0.55) - (rightHip?.y ?? 0.55));
|
|
const headOffset = Math.abs((nose?.x ?? 0.5) - (((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2));
|
|
const kneeBend = ((getAngle(leftHip, leftKnee, leftAnkle) || 165) + (getAngle(rightHip, rightKnee, rightAnkle) || 165)) / 2;
|
|
const rightElbowAngle = getAngle(rightShoulder, rightElbow, rightWrist) || 145;
|
|
const leftElbowAngle = getAngle(leftShoulder, leftElbow, leftWrist) || 145;
|
|
const footSpread = Math.abs((leftAnkle?.x ?? 0.42) - (rightAnkle?.x ?? 0.58));
|
|
const shoulderSpan = Math.abs((rightShoulder?.x ?? 0.56) - (leftShoulder?.x ?? 0.44));
|
|
const wristSpread = Math.abs((rightWrist?.x ?? 0.62) - (leftWrist?.x ?? 0.38));
|
|
const shoulderCenterX = ((leftShoulder?.x ?? 0.45) + (rightShoulder?.x ?? 0.55)) / 2;
|
|
const torsoOffset = Math.abs(shoulderCenterX - hipCenter.x);
|
|
const rightForward = (rightWrist?.x ?? shoulderCenterX) - hipCenter.x;
|
|
const leftForward = hipCenter.x - (leftWrist?.x ?? shoulderCenterX);
|
|
const contactHeight = hipCenter.y - (rightWrist?.y ?? hipCenter.y);
|
|
const landmarkVisibility = landmarks
|
|
.filter((item) => typeof item?.visibility === "number")
|
|
.map((item) => item.visibility as number);
|
|
const averageVisibility = landmarkVisibility.length > 0
|
|
? landmarkVisibility.reduce((sum, item) => sum + item, 0) / landmarkVisibility.length
|
|
: 0.8;
|
|
|
|
tracking.prevTimestamp = timestamp;
|
|
tracking.prevRightWrist = rightWrist;
|
|
tracking.prevLeftWrist = leftWrist;
|
|
tracking.prevHipCenter = hipCenter;
|
|
|
|
if (averageVisibility < 0.58 || shoulderSpan < 0.08 || footSpread < 0.05 || headOffset > 0.26) {
|
|
return { action: "unknown", confidence: 0.28 };
|
|
}
|
|
|
|
const serveConfidence = clamp(
|
|
rightVerticalMotion * 2.2 +
|
|
Math.max(0, (hipCenter.y - (rightWrist?.y ?? hipCenter.y)) * 3.4) +
|
|
(rightWrist?.y ?? 1) < (nose?.y ?? 0.3) ? 0.34 : 0 +
|
|
rightElbowAngle > 145 ? 0.12 : 0 -
|
|
shoulderTilt * 1.8,
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const overheadConfidence = clamp(
|
|
serveConfidence * 0.62 +
|
|
((rightWrist?.y ?? 1) < (nose?.y ?? 0.3) ? 0.22 : 0) +
|
|
(rightSpeed > 0.34 ? 0.16 : 0) -
|
|
(kneeBend < 150 ? 0.08 : 0),
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const forehandConfidence = clamp(
|
|
(rightSpeed * 1.5) +
|
|
Math.max(0, rightForward * 2.3) +
|
|
(rightElbowAngle > 120 ? 0.1 : 0) +
|
|
(hipSpeed > 0.07 ? 0.08 : 0) +
|
|
(footSpread > 0.12 ? 0.05 : 0) -
|
|
shoulderTilt * 1.1,
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const backhandConfidence = clamp(
|
|
(leftSpeed * 1.45) +
|
|
Math.max(0, leftForward * 2.15) +
|
|
(leftElbowAngle > 118 ? 0.1 : 0) +
|
|
(wristSpread > shoulderSpan * 1.2 ? 0.08 : 0) +
|
|
(torsoOffset > 0.04 ? 0.06 : 0),
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const volleyConfidence = clamp(
|
|
((rightSpeed + leftSpeed) * 0.8) +
|
|
(footSpread < 0.12 ? 0.12 : 0) +
|
|
(kneeBend < 155 ? 0.12 : 0) +
|
|
(Math.abs(contactHeight) < 0.16 ? 0.1 : 0) +
|
|
(hipSpeed > 0.08 ? 0.08 : 0),
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const sliceConfidence = clamp(
|
|
forehandConfidence * 0.68 +
|
|
((rightWrist?.y ?? 0.5) > hipCenter.y ? 0.12 : 0) +
|
|
(contactHeight < 0.05 ? 0.1 : 0),
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const lobConfidence = clamp(
|
|
overheadConfidence * 0.55 +
|
|
((rightWrist?.y ?? 1) < (leftShoulder?.y ?? 0.3) ? 0.14 : 0) +
|
|
(hipSpeed < 0.08 ? 0.06 : 0),
|
|
0,
|
|
1,
|
|
);
|
|
|
|
const candidates = ([
|
|
["serve", serveConfidence],
|
|
["overhead", overheadConfidence],
|
|
["forehand", forehandConfidence],
|
|
["backhand", backhandConfidence],
|
|
["volley", volleyConfidence],
|
|
["slice", sliceConfidence],
|
|
["lob", lobConfidence],
|
|
] as Array<[ActionType, number]>).sort((left, right) => right[1] - left[1]);
|
|
|
|
const [action, confidence] = candidates[0] || ["unknown", 0];
|
|
if (confidence < 0.45) {
|
|
return { action: "unknown", confidence: clamp(confidence, 0.18, 0.42) };
|
|
}
|
|
|
|
return { action, confidence: clamp(confidence, 0, 1) };
|
|
}
|
|
|
|
export function stabilizeActionFrame(frame: ActionFrame, history: ActionObservation[]) {
|
|
const nextHistory = [...history, { action: frame.action, confidence: frame.confidence }].slice(-6);
|
|
history.splice(0, history.length, ...nextHistory);
|
|
|
|
const weights = nextHistory.map((_, index) => index + 1);
|
|
const scores = nextHistory.reduce<Record<ActionType, number>>((acc, sample, index) => {
|
|
acc[sample.action] = (acc[sample.action] || 0) + sample.confidence * weights[index];
|
|
return acc;
|
|
}, {
|
|
forehand: 0,
|
|
backhand: 0,
|
|
serve: 0,
|
|
volley: 0,
|
|
overhead: 0,
|
|
slice: 0,
|
|
lob: 0,
|
|
unknown: 0,
|
|
});
|
|
|
|
const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1]) as Array<[ActionType, number]>;
|
|
const [winner = "unknown", winnerScore = 0] = ranked[0] || [];
|
|
const [, runnerScore = 0] = ranked[1] || [];
|
|
const winnerSamples = nextHistory.filter((sample) => sample.action === winner);
|
|
const averageConfidence = winnerSamples.length > 0
|
|
? winnerSamples.reduce((sum, sample) => sum + sample.confidence, 0) / winnerSamples.length
|
|
: frame.confidence;
|
|
|
|
const stableAction =
|
|
winner === "unknown" && frame.action !== "unknown" && frame.confidence >= 0.52
|
|
? frame.action
|
|
: winnerScore - runnerScore < 0.2 && frame.confidence >= 0.65
|
|
? frame.action
|
|
: winner;
|
|
|
|
return {
|
|
action: stableAction,
|
|
confidence: clamp(stableAction === frame.action ? Math.max(frame.confidence, averageConfidence) : averageConfidence, 0, 1),
|
|
};
|
|
}
|