Nhat Dinh Nhat Dinh - 1 month ago 29
iOS Question

Convert dual fisheye video to equirectangular

I used open framework to convert dual fisheye video (received from ricoh theta S camera) to equirectangular format (for an iOS application).

dual fisheye image sample:
enter image description here

equirectangular image sample:
enter image description here

I use below shader:

equirectangular.frag


// based on ThetaShaderPack_20150926 (http://stereoarts.jp/) written by Nora.
#ifdef GL_ES
// define default precision for float, vec, mat.
precision highp float;
#endif

#define PI 3.14159265358979
#define _THETA_S_Y_SCALE (640.0 / 720.0)

uniform sampler2D mainTex;
uniform float radius;
uniform vec4 uvOffset;

varying vec2 texCoordVarying;

void main (void) {
vec2 revUV = texCoordVarying.st;
if (texCoordVarying.x <= 0.5) {
revUV.x = revUV.x * 2.0;
} else {
revUV.x = (revUV.x - 0.5) * 2.0;
}

revUV *= PI;

vec3 p = vec3(cos(revUV.x), cos(revUV.y), sin(revUV.x));
p.xz *= sqrt(1.0 - p.y * p.y);

float r = 1.0 - asin(p.z) / (PI / 2.0);
vec2 st = vec2(p.y, p.x);

st *= r / sqrt(1.0 - p.z * p.z);
st *= radius;
st += 0.5;

if (texCoordVarying.x <= 0.5) {
st.x *= 0.5;
st.x += 0.5;
st.y = 1.0 - st.y;
st.xy += uvOffset.wz;
} else {
st.x = 1.0 - st.x;
st.x *= 0.5;
st.xy += uvOffset.yx;
}

st.y = st.y * _THETA_S_Y_SCALE;

gl_FragColor = texture2D(mainTex, st);
}


equirectanguler.vert


uniform mat4 projectionMatrix;
uniform mat4 modelViewMatrix;
uniform mat4 textureMatrix;
uniform mat4 modelViewProjectionMatrix;

attribute vec4 position;
attribute vec4 color;
attribute vec3 normal;
attribute vec2 texcoord;

varying vec2 texCoordVarying;

void main() {
texCoordVarying = texcoord;
gl_Position = modelViewProjectionMatrix * position;
}


with below code:
main.mm


#include "ofApp.h"

int main() {

// here are the most commonly used iOS window settings.
//------------------------------------------------------
ofiOSWindowSettings settings;
settings.enableRetina = false; // enables retina resolution if the device supports it.
settings.enableDepth = false; // enables depth buffer for 3d drawing.
settings.enableAntiAliasing = false; // enables anti-aliasing which smooths out graphics on the screen.
settings.numOfAntiAliasingSamples = 0; // number of samples used for anti-aliasing.
settings.enableHardwareOrientation = false; // enables native view orientation.
settings.enableHardwareOrientationAnimation = false; // enables native orientation changes to be animated.
settings.glesVersion = OFXIOS_RENDERER_ES2; // type of renderer to use, ES1, ES2, ES3
settings.windowMode = OF_FULLSCREEN;
ofCreateWindow(settings);

return ofRunApp(new ofApp);
}

```

`offApp.mm`

```
#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){
ofDisableArbTex();

devices = theta.listDevices();
bool isDeviceConnected = false;
for(int i = 0; i < devices.size(); i++){
if(devices[i].deviceName == "RICOH THETA S"){
theta.setDeviceID(devices[i].id);
isDeviceConnected = true;
}
}
if(!isDeviceConnected){
ofLog(OF_LOG_ERROR, "RICOH THETA S is not found.");
}
theta.initGrabber(360, 568);

shader.load("shaders/equirectanguler");

fbo.allocate(320, 568);

sphere = ofSpherePrimitive(568, 64).getMesh();
for(int i=0;i<sphere.getNumTexCoords();i++){
sphere.setTexCoord(i, ofVec2f(1.0) - sphere.getTexCoord(i));
}
for(int i=0;i<sphere.getNumNormals();i++){
sphere.setNormal(i, sphere.getNormal(i) * ofVec3f(-1));
}

offset.set("uvOffset", ofVec4f(0,0.0,0,0.0), ofVec4f(-0.1), ofVec4f(0.1));
radius.set("radius", 0.445, 0.0, 1.0);
showSphere.set("showSphere", false);
thetaParams.add(offset);
thetaParams.add(radius);
gui.setup(thetaParams);
gui.add(showSphere);

cam.setAutoDistance(false);
cam.setDistance(0);
}

//--------------------------------------------------------------
void ofApp::update(){
theta.update();
}

//--------------------------------------------------------------
void ofApp::draw(){
if(theta.isFrameNew()){

fbo.begin();
ofClear(0);
shader.begin();
shader.setUniformTexture("mainTex", theta.getTexture(), 0);
shader.setUniforms(thetaParams);
theta.draw(0, 0, 320, 568);
shader.end();
fbo.end();

}

if(!showSphere){

fbo.draw(0, 0, 320, 568);

}else{

ofEnableDepthTest();
cam.begin();
fbo.getTexture().bind();
sphere.draw();
fbo.getTexture().unbind();
cam.end();

}

ofDisableDepthTest();
gui.draw();
}

//--------------------------------------------------------------
void ofApp::exit(){

}

//--------------------------------------------------------------
void ofApp::touchDown(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchMoved(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchUp(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchDoubleTap(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchCancelled(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::lostFocus(){

}

//--------------------------------------------------------------
void ofApp::gotFocus(){

}

//--------------------------------------------------------------
void ofApp::gotMemoryWarning(){

}

//--------------------------------------------------------------
void ofApp::deviceOrientationChanged(int newOrientation){

}


offApp.h


#pragma once

#include "ofxiOS.h"
#include "ofxGui.h"

class ofApp : public ofxiOSApp {

public:
void setup();
void update();
void draw();
void exit();

void touchDown(ofTouchEventArgs & touch);
void touchMoved(ofTouchEventArgs & touch);
void touchUp(ofTouchEventArgs & touch);
void touchDoubleTap(ofTouchEventArgs & touch);
void touchCancelled(ofTouchEventArgs & touch);

void lostFocus();
void gotFocus();
void gotMemoryWarning();
void deviceOrientationChanged(int newOrientation);

ofVideoGrabber theta;
vector<ofVideoDevice> devices;
ofShader shader;
ofFbo fbo;
ofEasyCam cam;
ofVboMesh sphere;

ofParameter<ofVec4f> offset;
ofParameter<float> radius;
ofParameter<bool> showSphere;
ofParameterGroup thetaParams;
ofxPanel gui;
};


And here is the result:
on iphone5s
enter image description here

pls point me out what i missed.

Answer

I switch from OpenFrameworks to GPUImage and be able to convert dual-fisheye video to equirectangular format successfully on iOS devices.

Used same shader program. check my repository

Comments