Voice Controlled Cube

Troubleshooting android
1

I have managed to get the voice recognition component of android to send commands into jme.



[java]package jme3.innovationtech.co.uk;



import java.util.ArrayList;



import android.content.Intent;

import android.os.Bundle;

import android.speech.RecognizerIntent;

import android.util.Log;

import android.view.View;

import android.widget.FrameLayout;

import android.widget.Toast;



import com.jme3.app.AndroidHarness;

import com.jme3.system.android.AndroidConfigChooser.ConfigType;



public class MainActivity extends AndroidHarness {



Game3 game;

private static final int REQUEST_CODE = 1234;



@Override

protected void onResume() {

super.onResume();

}



@Override

protected void onStop() {

super.onStop();

}



@Override

public void onCreate(Bundle savedInstanceState)

{

super.onCreate(savedInstanceState);

setContentView(R.layout.main);

FrameLayout frame = (FrameLayout) findViewById(R.id.threeD_view);

frame.addView(view);

game=(Game3) getJmeApplication();



}



public MainActivity() {

//appClass = "jme3.innovationtech.co.uk.Game";

appClass = "jme3.innovationtech.co.uk.Game3";

eglConfigType = ConfigType.BEST;



mouseEventsEnabled=false;

exitDialogTitle = "Exit?";

exitDialogMessage = "Press Yes";

eglConfigVerboseLogging = false;

}



/**

  • Handle the action of the button being clicked

    */

    public void speakButtonClicked(View v)

    {

    startVoiceRecognitionActivity();

    }



    /**
  • Fire an intent to start the voice recognition activity.

    */

    private void startVoiceRecognitionActivity()

    {

    Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);

    intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,

    RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);

    intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Voice recognition Demo…");

    startActivityForResult(intent, REQUEST_CODE);

    }



    /**
  • Handle the results from the voice recognition activity.

    /

    @Override

    protected void onActivityResult(int requestCode, int resultCode, Intent data)

    {

    if (requestCode == REQUEST_CODE && resultCode == RESULT_OK)

    {

    // Populate the wordsList with the String values the recognition engine thought it heard

    ArrayList<String> matches = data.getStringArrayListExtra(

    RecognizerIntent.EXTRA_RESULTS);



    String[] strings = matches.toArray(new String[0]);

    String command=strings[0];



    Toast.makeText(this, "You selected: " + command, Toast.LENGTH_LONG).show();

    if (command.equals("move")){

    game.animate=true;

    }

    if (command.equals("stop")){

    game.animate=false;

    }

    }

    super.onActivityResult(requestCode, resultCode, data);

    }

    }[/java]



    [java]package jme3.innovationtech.co.uk;



    // loaded dancing mesh

    // long press spins

    // one click changes animation



    import android.util.Log;



    import com.jme3.animation.AnimChannel;

    import com.jme3.animation.AnimControl;

    import com.jme3.animation.AnimEventListener;

    import com.jme3.animation.LoopMode;

    import com.jme3.app.SimpleApplication;

    import com.jme3.asset.AssetManager;

    import com.jme3.asset.TextureKey;

    import com.jme3.bullet.BulletAppState;

    import com.jme3.bullet.PhysicsSpace;

    import com.jme3.bullet.collision.PhysicsCollisionEvent;

    import com.jme3.bullet.collision.PhysicsCollisionListener;

    import com.jme3.bullet.collision.shapes.CapsuleCollisionShape;

    import com.jme3.bullet.collision.shapes.SphereCollisionShape;

    import com.jme3.bullet.control.CharacterControl;

    import com.jme3.bullet.control.RigidBodyControl;

    import com.jme3.collision.CollisionResult;

    import com.jme3.collision.CollisionResults;

    import com.jme3.effect.ParticleEmitter;

    import com.jme3.font.BitmapText;

    import com.jme3.input.ChaseCamera;

    import com.jme3.input.KeyInput;

    import com.jme3.input.MouseInput;

    import com.jme3.input.controls.ActionListener;

    import com.jme3.input.controls.KeyTrigger;

    import com.jme3.input.controls.MouseButtonTrigger;

    import com.jme3.input.controls.TouchListener;

    import com.jme3.input.controls.TouchTrigger;

    import com.jme3.input.event.TouchEvent;

    import com.jme3.light.AmbientLight;

    import com.jme3.light.DirectionalLight;

    import com.jme3.material.Material;

    import com.jme3.math.ColorRGBA;

    import com.jme3.math.Matrix3f;

    import com.jme3.math.Ray;

    import com.jme3.math.Vector2f;

    import com.jme3.math.Vector3f;

    import com.jme3.renderer.Renderer;

    import com.jme3.renderer.queue.RenderQueue.ShadowMode;

    import com.jme3.scene.Geometry;

    import com.jme3.scene.Node;

    import com.jme3.scene.Spatial;

    import com.jme3.scene.shape.Box;

    import com.jme3.scene.shape.Quad;

    import com.jme3.scene.shape.Sphere;

    import com.jme3.scene.shape.Sphere.TextureMode;

    import com.jme3.texture.Texture;

    import com.jme3.texture.Texture.WrapMode;

    import com.jme3.ui.Picture;

    import com.jme3.util.SkyFactory;



    import com.jme3.effect.ParticleMesh;



    public class Game3 extends SimpleApplication {



    protected Geometry player;

    public boolean animate=true;



    @Override

    public void simpleInitApp() {



    Box b = new Box(Vector3f.ZERO, 1, 1, 1);

    player = new Geometry("blue cube", b);

    Material mat = new Material(assetManager,

    "Common/MatDefs/Misc/Unshaded.j3md");

    mat.setColor("Color", ColorRGBA.Blue);

    player.setMaterial(mat);

    rootNode.attachChild(player);

    }



    /     This is the update loop /

    @Override

    public void simpleUpdate(float tpf) {

    // make the player rotate

    if (animate){

    player.rotate(0, 2    tpf, 0);

    }

    }

    }[/java]



    [xml]<?xml version="1.0" encoding="utf-8"?>

    <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"

    android:orientation="vertical"

    android:layout_width="fill_parent"

    android:layout_height="fill_parent"

    >

    <Button android:id="@+id/speakButton"

    android:layout_width="fill_parent"

    android:onClick="speakButtonClicked"

    android:layout_height="wrap_content"

    android:text="Voice Command!" />



    <FrameLayout

    android:id="@+id/threeD_view"

    android:layout_height="match_parent"

    android:layout_width="match_parent"></FrameLayout>



    </LinearLayout>

    [/xml]



    The trick is to encapsulate the glsurface or jme3 instance in a frame layout. This technique will also be useful for integrating into other existing android apps which use the android UI and its available components. (IE you can add a 3d component/widget)
5 Likes
2

Ok. This is cool. I may have to integrate this into my project. Thanks!

3

@mikegriffin you’re becoming an Android all-star, this is really cool! Any chance you could make a little demo video?

4

Its very nice. Just one thing, in the SDK the Application and Harness part are stictly separated and the Application class has no access to the android api (as its supposed to be platform independent). People will have an easier time trying your code in the SDK if you’d use the normal java logging inside the Application class.

Thanks very much for sharing your code, I’m sure you’re helping way more people with these posts than you realize :slight_smile:

5

I also didn’t like how the game was forced to be full screen. On the phone, I like to have the notification bar visible at the top while the game is playing. That way, for example, when a text message comes in, the user can still pull down the notification bar to get to the text quickly. Also, when the text app is closed, you’re right back to the game. A while ago, I asked for a change to AndroidHarness that allowed AndroidHarness to size the GLSurfaceView to allow for the notification bar to be visible. However, the side effect is that the game is stil being started with the settings.height and setting.width to be the full screen size until the surfaceview is created which resizes the settings appropriately. This is sometime after simpleInitApp but before simpleUpdate which forces me to draw my guiNode items during the first scan of simpleUpdate instead of placing them in simpleInitApp. No big deal, to me anyway.



Did you notice if putting the surfaceview into a frame layout also has the same effect?

6

@iwgeric - I haven’t really explored this in detail. At the moment I’m just playing around to see what can be accomplished with the engine and android os.

7

@sbook - I’m trying to put together a number of basic demos into an apk which I’ll release along with the entire eclipse project and source code.

8

@normen - thanks for the comment and positive feedback



I understand the Application class doesn’t have access to the Android OS and probably doesn’t even need to know about it or its capabilities. I’m just approaching it really objectively. Introducing a harness into the mix abstracts the JME3 from its parent OS. The Harness has a series of inputs from the OS/device which can be mapped to the Application.



The Harness itself is an extension of the activity class and has all the hooks into the Android OS. The harness in effect becomes the Controller - taking input and output from the OS or the Application class. I’m basically extending the harness to do more and punching through some mechanisms for the Harness to manipulate the Application by extending it as necessary. At the moment I’m using it as a one way process - I’m not trying to take output from the Application - though in effect you could (for multiplayer updates, sms, vibration alerts, communicating and leveraging other activities/apps in the android os etc)



I agree it does muddy the water a little about where to put the application logic and the core control of the application.