prendy
Version:
Make games with prerendered backdrops using babylonjs and repond
239 lines (238 loc) • 12 kB
JavaScript
import { FxaaPostProcess, PBRMaterial, PostProcess, ShaderStore } from "@babylonjs/core";
import { chooseClosestBeforeItemInArray } from "chootils/dist/arrays";
import { forEach } from "chootils/dist/loops";
import { getRefs, getState, setState } from "repond";
import { meta } from "../../meta";
import shaders from "../shaders";
import { getGlobalState } from "./global";
import { getSegmentFromSegmentRules } from "./scene";
import { focusSlateOnFocusedDoll } from "../../helpers/babylonjs/slate";
import { getNowBackdropFrameInfo } from "../../helpers/prendyUtils/backdrops";
export function getSafeCamName(cam) {
if (cam === null)
return null;
const { placeInfoByName } = meta.assets;
const { nowPlaceName } = getGlobalState();
const safePlace = nowPlaceName;
const { segmentNamesByCamera, cameraNames } = placeInfoByName[safePlace];
// if the camera isn't in the nowPlace, then use the first camera for the nowPlace
const safeCam = segmentNamesByCamera?.[cam] ? cam : cameraNames[0];
return safeCam;
}
export function getSafeSegmentName({ cam, place, segment, useStorySegmentRules, }) {
const { placeInfoByName, dollNames } = meta.assets;
const { nowPlaceName } = getGlobalState();
const safePlace = nowPlaceName;
const { segmentNames, segmentNamesByCamera } = placeInfoByName[safePlace];
const safeCam = getSafeCamName(cam);
if (nowPlaceName !== place) {
console.warn("tried to getSafeSegment name for not current place", place);
}
const camSegmentNames = segmentNamesByCamera?.[safeCam] ?? {};
// disabling for now to allow getSafeSegmentName to work in video.ts (looping stuff) when changing segment?
const foundRuleSegmentName = useStorySegmentRules
? getSegmentFromSegmentRules(safePlace, safeCam)
: undefined;
return chooseClosestBeforeItemInArray({
fullArray: segmentNames,
goalItem: foundRuleSegmentName ?? segment,
smallArray: camSegmentNames,
});
}
// Updates both backdrop textures and probe textures for the new cameras
export function updateTexturesForNowCamera(newCameraName, didChangePlace = false) {
focusSlateOnFocusedDoll("instant");
const globalRefs = getRefs().global.main;
const placesRefs = getRefs().places;
const { nowPlaceName, nowSegmentName } = getState().global.main;
const scene = globalRefs.scene;
const placeRef = placesRefs[nowPlaceName];
const { camsRefs } = placeRef;
const newCamRef = camsRefs[newCameraName];
if (scene === null)
return;
if (!newCamRef.camera)
return;
// Render target
if (globalRefs.backdropPostProcess)
scene.activeCamera?.detachPostProcess(globalRefs.backdropPostProcess);
if (globalRefs.fxaaPostProcess)
scene.activeCamera?.detachPostProcess(globalRefs.fxaaPostProcess);
scene.activeCamera = newCamRef.camera;
if (globalRefs.backdropPostProcess)
scene.activeCamera?.attachPostProcess(globalRefs.backdropPostProcess);
if (globalRefs.fxaaPostProcess)
scene.activeCamera?.attachPostProcess(globalRefs.fxaaPostProcess);
// Create the depth renderer
if (!globalRefs.depthRenderer)
globalRefs.depthRenderer = scene.enableDepthRenderer(newCamRef.camera, false);
// Update the depth renderer camera, without disposing it
// @ts-ignore
globalRefs.depthRenderer._camera = newCamRef.camera;
// Create the depth render target
if (!globalRefs.depthRenderTarget)
globalRefs.depthRenderTarget = globalRefs.depthRenderer.getDepthMap();
// Update the depth render target camera
globalRefs.depthRenderTarget.activeCamera = newCamRef.camera;
if (!scene.customRenderTargets.length) {
scene.customRenderTargets = [globalRefs.depthRenderTarget];
addMeshesToRenderLists(newCamRef);
}
if (scene.activeCamera && !globalRefs.backdropPostProcess) {
ShaderStore.ShadersStore["depthyPixelShader"] = shaders.backdropAndDepth.backdropFragment;
ShaderStore.ShadersStore["depthyVertexShader"] = shaders.backdropAndDepth.backdropVertex;
globalRefs.backdropPostProcess = new PostProcess("backdropAndDepthShader", "depthy", [
"slatePos",
"stretchSceneAmount",
"stretchVideoAmount",
"currentFrameIndex",
"framesPerRow",
"framesPerColumn",
"frameSize",
], [
"textureSampler",
"SceneDepthTexture",
"BackdropTextureSample",
"BackdropColorTextureSample",
"BackdropDepthTextureSample",
], // textures
1, scene.activeCamera, undefined, undefined, undefined, undefined, undefined, "depthy");
updateVideoTexture();
// focusSlateOnFocusedDoll("instant");
// NOTE this is only needed for the first time the camera loads, it should probably be somewhere else
setTimeout(() => {
setState({ global: { main: { timeScreenResized: Date.now() } } });
}, 10);
}
// Set the onApply for the post process
if (!globalRefs.backdropPostProcess.onApply) {
globalRefs.backdropPostProcess.onApply = (effect) => {
let newCamera = getState().global.main.nowCamName;
let camRef = placesRefs[nowPlaceName].camsRefs[newCamera];
// TODO move this to calcuate once when changing camera/segment, and save it in global refs or state
const { frameSize, framesPerColumn, framesPerRow, nowTextureIndex, backdropFrameForNowTexture } = getNowBackdropFrameInfo(newCamera);
updateVideoTexture();
const { slatePos, slatePosGoal, slateZoom } = getState().global.main;
// There can be multiple atlas textures, so we need to calculate the correct frame for the current texture
// And also which texture should be used, based on the current frame and the max frames per texture
// update the active texture
if (!globalRefs.backdropPostProcessEffect) {
globalRefs.backdropPostProcessEffect = effect;
effect.setFloat2("slatePos", slatePosGoal.x, slatePosGoal.y);
effect.setFloat2("stretchSceneAmount", slateZoom, slateZoom);
effect.setFloat2("stretchVideoAmount", 1, 1);
}
effect.setFloat("currentFrameIndex", backdropFrameForNowTexture);
effect.setFloat("framesPerRow", framesPerRow);
effect.setFloat("framesPerColumn", framesPerColumn);
effect.setVector2("frameSize", frameSize);
globalRefs.backdropTex = camRef.backdropTexturesBySegment[nowSegmentName][nowTextureIndex].color;
globalRefs.backdropTexDepth = camRef.backdropTexturesBySegment[nowSegmentName][nowTextureIndex].depth;
globalRefs?.backdropPostProcessEffect?.setTexture("BackdropColorTextureSample", globalRefs.backdropTex);
globalRefs?.backdropPostProcessEffect?.setTexture("BackdropDepthTextureSample", globalRefs.backdropTexDepth);
globalRefs?.backdropPostProcessEffect?.setFloat2("stretchVideoAmount", globalRefs.stretchVideoSize.x, globalRefs.stretchVideoSize.y);
globalRefs?.backdropPostProcessEffect?.setFloat2("stretchSceneAmount", globalRefs.stretchSceneSize.x, globalRefs.stretchSceneSize.y);
const positionChanged = true;
const zoomChanged = true;
const engine = scene?.getEngine(); // engine
if (engine && (positionChanged || zoomChanged)) {
const stretchVideoSize = globalRefs.stretchVideoSize;
globalRefs?.backdropPostProcessEffect?.setFloat2("slatePos", slatePos.x * stretchVideoSize.x, slatePos.y * stretchVideoSize.y);
}
};
}
// Make the FXAA post process
if (scene.activeCamera && !globalRefs.fxaaPostProcess) {
globalRefs.fxaaPostProcess = new FxaaPostProcess("fxaa", 1.0, scene.activeCamera);
}
if (didChangePlace) {
addMeshesToRenderLists(newCamRef);
updateVideoTexture();
}
applyProbeToAllDollMaterials();
applyProbeToAllParticleMaterials();
}
export function addMeshesToRenderLists(newCamRef) {
const { dollNames } = meta.assets;
const globalRefs = getRefs().global.main;
const scene = globalRefs.scene;
// scene.freeActiveMeshes(); // hm? different to freezeActiveMeshes , maybe unintentional
if (globalRefs.depthRenderTarget)
globalRefs.depthRenderTarget.renderList = [];
forEach(dollNames, (dollName) => {
const dollMeshes = getRefs().dolls[dollName].otherMeshes;
const dollMeshNames = Object.keys(dollMeshes);
forEach(dollMeshNames, (meshName) => {
const dollMesh = dollMeshes[meshName];
if (dollMesh) {
dollMesh.isInFrustum = () => true;
globalRefs.depthRenderTarget?.renderList?.push(dollMesh);
}
});
});
const particleSystemNames = Object.keys(globalRefs.solidParticleSystems);
forEach(particleSystemNames, (particleSystemName) => {
const particleSystem = globalRefs.solidParticleSystems[particleSystemName];
globalRefs.depthRenderTarget?.renderList?.push(particleSystem.mesh);
particleSystem._camera = newCamRef.camera;
});
// @ts-ignore
scene._skipEvaluateActiveMeshesCompletely = true;
}
export function updateVideoTexture() {
const globalRefs = getRefs().global.main;
globalRefs?.backdropPostProcessEffect?.setTexture("BackdropColorTextureSample", globalRefs.backdropTex);
globalRefs?.backdropPostProcessEffect?.setTexture("BackdropDepthTextureSample", globalRefs.backdropTexDepth);
globalRefs?.backdropPostProcessEffect?.setTexture("SceneDepthTexture", globalRefs.depthRenderTarget);
}
export function applyProbeToAllDollMaterials() {
const { dollNames } = meta.assets;
const globalRefs = getRefs().global.main;
const placesRefs = getRefs().places;
const { nowPlaceName, modelNamesLoaded } = getState().global.main;
const { nowCamName } = getState().global.main;
const { scene } = globalRefs;
const placeRef = placesRefs[nowPlaceName];
const { camsRefs } = placeRef;
const newCamRef = camsRefs[nowCamName];
if (scene === null)
return;
forEach(modelNamesLoaded, (modelName) => {
const modelRefs = getRefs().models[modelName];
if (modelRefs.materialRef && newCamRef.probeTexture) {
modelRefs.materialRef.reflectionTexture = newCamRef.probeTexture;
}
});
// // looks like the dolls material doesn't automatically update with the models material
// // so setting it here works :)
forEach(dollNames, (dollName) => {
const dollRefs = getRefs().dolls[dollName];
const { modelName } = getState().dolls[dollName];
const modelRefs = getRefs().models[modelName];
if (dollRefs.meshRef) {
dollRefs.meshRef.material = modelRefs.materialRef;
dollRefs.meshRef.material?.freeze();
}
});
}
export function applyProbeToAllParticleMaterials() {
const globalRefs = getRefs().global.main;
const placesRefs = getRefs().places;
const { nowPlaceName } = getState().global.main;
const { nowCamName } = getState().global.main;
const { scene } = globalRefs;
const placeRef = placesRefs[nowPlaceName];
const { camsRefs } = placeRef;
const newCamRef = camsRefs[nowCamName];
if (scene === null)
return;
const particleSystemNames = Object.keys(globalRefs.solidParticleSystems);
particleSystemNames.forEach((particleSystemName) => {
const particleSystem = globalRefs.solidParticleSystems[particleSystemName];
const material = particleSystem.mesh.material;
if (material && material instanceof PBRMaterial && newCamRef.probeTexture) {
material.reflectionTexture = newCamRef.probeTexture;
}
globalRefs.depthRenderTarget?.renderList?.push(particleSystem.mesh);
});
}