Skip to content

Commit a80c15b

Browse files
author
Ankush Agarwal
authored
Merge pull request kubeflow#213 from activatedgeek/search-server-kubeflow
Update Search Index server spec
2 parents cfdcb12 + 6e9150b commit a80c15b

File tree

10 files changed

+150
-55
lines changed

10 files changed

+150
-55
lines changed

‎code_search/docker/t2t/build.sh‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,14 @@
88

99
set -ex
1010

11+
PROJECT=${PROJECT:-}
1112
BUILD_IMAGE_UUID=$(python3 -c 'import uuid; print(uuid.uuid4().hex[:7]);')
1213
BUILD_IMAGE_TAG="code-search:v$(date +%Y%m%d)-${BUILD_IMAGE_UUID}"
1314

15+
if [[ ! -z "${PROJECT}" ]]; then
16+
BUILD_IMAGE_TAG="gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}"
17+
fi
18+
1419
# Directory of this script used for path references
1520
_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
1621

@@ -29,15 +34,9 @@ docker build -f "${_SCRIPT_DIR}/Dockerfile" \
2934
"${_SCRIPT_DIR}/../.."
3035

3136
# Push images to GCR Project if available
32-
PROJECT=${PROJECT:-}
3337
if [[ ! -z "${PROJECT}" ]]; then
34-
# Tag and push CPU image
35-
docker tag ${BUILD_IMAGE_TAG} gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}
36-
docker push gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}
37-
38-
# Tag and push GPU image
39-
docker tag ${BUILD_IMAGE_TAG}-gpu gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}-gpu
40-
docker push gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}-gpu
38+
docker push ${BUILD_IMAGE_TAG}
39+
docker push ${BUILD_IMAGE_TAG}-gpu
4140
fi
4241

4342
popd

‎code_search/docker/ui/Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ ADD src/ /src
1616

1717
WORKDIR /src
1818

19+
ARG PUBLIC_URL
20+
21+
ENV PUBLIC_URL=$PUBLIC_URL
22+
1923
RUN cd ui && npm i && npm run build && cd ..
2024

2125
EXPOSE 8008

‎code_search/docker/ui/build.sh‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,36 @@
55
# the Code Search UI to Google Container Registry. It automatically tags
66
# a unique image for every run.
77
#
8+
# Also note the PUBLIC_URL endpoint which can either be an FQDN or a path
9+
# prefix to be used for the JS files in the React Application. It defaults
10+
# to "/code-search" which means the search server will be accessible at
11+
# https://PUBLIC_HOSTNAME/code-search.
12+
#
813

914
set -ex
1015

16+
PROJECT=${PROJECT:-}
1117
BUILD_IMAGE_UUID=$(python3 -c 'import uuid; print(uuid.uuid4().hex[:7]);')
1218
BUILD_IMAGE_TAG="code-search-ui:v$(date +%Y%m%d)-${BUILD_IMAGE_UUID}"
19+
PUBLIC_URL=${PUBLIC_URL:-"/code-search"}
20+
21+
if [[ ! -z "${PROJECT}" ]]; then
22+
BUILD_IMAGE_TAG="gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}"
23+
fi
1324

1425
# Directory of this script used for path references
1526
_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
1627

1728
pushd "${_SCRIPT_DIR}"
1829

19-
docker build -f "${_SCRIPT_DIR}/Dockerfile" -t ${BUILD_IMAGE_TAG} "${_SCRIPT_DIR}/../.."
30+
docker build -f "${_SCRIPT_DIR}/Dockerfile" \
31+
-t ${BUILD_IMAGE_TAG} \
32+
--build-arg PUBLIC_URL=${PUBLIC_URL} \
33+
"${_SCRIPT_DIR}/../.."
2034

21-
# Push image to GCR PROJECT available
22-
PROJECT=${PROJECT:-}
35+
# Push images to GCR Project if available
2336
if [[ ! -z "${PROJECT}" ]]; then
24-
docker tag ${BUILD_IMAGE_TAG} gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}
25-
docker push gcr.io/${PROJECT}/${BUILD_IMAGE_TAG}
37+
docker push ${BUILD_IMAGE_TAG}
2638
fi
2739

2840
popd

‎code_search/kubeflow/components/nms.libsonnet‎

Lines changed: 93 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
local baseParams = std.extVar("__ksonnet/params").components["nmslib"];
22

33
{
4-
5-
nmsContainer(params, env):: {
4+
deploymentSpec(params, env, containers, volumes=[]):: {
65
apiVersion: "extensions/v1beta1",
76
kind: "Deployment",
87
metadata: {
9-
name: params.name + "-deployment",
8+
name: params.name,
109
namespace: env.namespace,
1110
labels: {
1211
app: params.name,
@@ -26,23 +25,49 @@ local baseParams = std.extVar("__ksonnet/params").components["nmslib"];
2625
}
2726
},
2827
spec: {
29-
containers: [
30-
{
31-
name: params.name,
32-
image: params.image,
33-
args: params.args,
34-
ports: [
35-
{
36-
containerPort: 8008,
37-
}
38-
],
39-
}
40-
],
28+
containers: containers,
29+
volumes: volumes,
4130
},
4231
},
4332
},
4433
},
4534

35+
jobSpec(params, env, containers, volumes=[]):: {
36+
apiVersion: "batch/v1",
37+
kind: "Job",
38+
metadata: {
39+
name: params.name,
40+
namespace: env.namespace,
41+
labels: {
42+
app: params.name,
43+
}
44+
},
45+
spec: {
46+
replicas: params.replicas,
47+
template: {
48+
metadata: {
49+
labels: {
50+
app: params.name,
51+
}
52+
},
53+
spec: {
54+
"restartPolicy": "OnFailure",
55+
containers: containers,
56+
volumes: volumes,
57+
},
58+
},
59+
},
60+
},
61+
62+
containerSpec(params, env=[], volumeMounts=[], ports=[]):: {
63+
name: params.name,
64+
image: params.image,
65+
args: params.args,
66+
ports: ports,
67+
env: env,
68+
volumeMounts: volumeMounts,
69+
},
70+
4671
service(params, env):: {
4772
apiVersion: "v1",
4873
kind: "Service",
@@ -84,35 +109,78 @@ local baseParams = std.extVar("__ksonnet/params").components["nmslib"];
84109
parts(newParams, env):: {
85110
local params = baseParams + newParams,
86111

112+
local volumes = [
113+
{
114+
name: "gcp-credentials",
115+
secret: {
116+
secretName: "user-gcp-sa",
117+
},
118+
},
119+
],
120+
121+
local containerEnv = [
122+
{
123+
name: "GOOGLE_APPLICATION_CREDENTIALS",
124+
value: "/secret/gcp-credentials/user-gcp-sa.json",
125+
}
126+
],
127+
128+
local containerVolumeMounts = [
129+
{
130+
mountPath: "/secret/gcp-credentials",
131+
name: "gcp-credentials",
132+
},
133+
],
134+
87135
creator:: {
88136
local creatorParams = params + {
89137
args: [
90-
"nmslib-create",
91-
"--data-file=" + params.dataFile,
92-
"--index-file=" + params.indexFile,
138+
"-m",
139+
"code_search.nmslib.cli.create_search_index",
140+
"--data_dir=" + params.dataDir,
141+
"--lookup_file=" + params.lookupFile,
142+
"--index_file=" + params.indexFile,
93143
],
94144
},
95145

96146
all: [
97-
$.nmsContainer(creatorParams, env),
147+
$.jobSpec(creatorParams, env,
148+
[
149+
$.containerSpec(creatorParams, env=containerEnv,
150+
volumeMounts=containerVolumeMounts)
151+
],
152+
volumes=volumes),
98153
],
99154
}.all,
100155

101156
server:: {
102157
local serverParams = params + {
103158
args: [
104-
"nmslib-serve",
105-
"--data-file=" + params.dataFile,
106-
"--index-file=" + params.indexFile,
159+
"-m",
160+
"code_search.nmslib.cli.start_search_server",
107161
"--problem=" + params.problem,
108-
"--data-dir=" + params.dataDir,
109-
"--serving-url=" + params.servingUrl,
162+
"--data_dir=" + params.dataDir,
163+
"--lookup_file=" + params.lookupFile,
164+
"--index_file=" + params.indexFile,
165+
"--serving_url=" + params.servingUrl,
110166
],
111167
},
112168

169+
local containerPorts = [
170+
{
171+
containerPort: 8008,
172+
}
173+
],
174+
113175
all: [
114176
$.service(serverParams, env),
115-
$.nmsContainer(serverParams, env),
177+
$.deploymentSpec(serverParams, env,
178+
[
179+
$.containerSpec(serverParams, env=containerEnv,
180+
volumeMounts=containerVolumeMounts,
181+
ports=containerPorts)
182+
],
183+
volumes=volumes),
116184
],
117185
}.all,
118186
}

‎code_search/kubeflow/components/params.libsonnet‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -68,27 +68,36 @@
6868
modelPath: $.global.t2tWorkingDir + "/output/export/Servo",
6969
modelServerImage: "gcr.io/kubeflow-images-public/tensorflow-serving-1.8:latest",
7070
cloud: "gcp",
71-
gcpCredentialSecretName: "gcp-credentials",
71+
gcpCredentialSecretName: "user-gcp-sa",
7272
},
7373

7474
"nmslib": {
75-
name: null,
7675
replicas: 1,
77-
image: "gcr.io/kubeflow-dev/code-search:v20180621-266e689",
76+
image: "gcr.io/kubeflow-dev/code-search-ui:v20180806-7b0fcaa",
7877

79-
dataFile: null,
80-
indexFile: null,
81-
problem: null,
82-
dataDir: null,
83-
servingUrl: null,
78+
problem: "null",
79+
dataDir: "null",
80+
lookupFile: "null",
81+
indexFile: "null",
82+
servingUrl: "null",
8483
},
8584

86-
"nms-creator": {
87-
name: "nms-creator",
85+
"search-index-creator": {
86+
name: "search-index-creator",
87+
88+
dataDir: $.global.t2tWorkingDir + "/data",
89+
lookupFile: $.global.t2tWorkingDir + "/code_search_index.csv",
90+
indexFile: $.global.t2tWorkingDir + "/code_search_index.nmslib",
8891
},
8992

90-
"nms-server": {
91-
name: "nms-server",
93+
"search-index-server": {
94+
name: "search-index-server",
95+
96+
problem: "github_function_docstring",
97+
dataDir: $.global.t2tWorkingDir + "/data",
98+
lookupFile: $.global.t2tWorkingDir + "/code_search_index.csv",
99+
indexFile: $.global.t2tWorkingDir + "/code_search_index.nmslib",
100+
servingUrl: "http://t2t-code-search.kubeflow:8000/v1/models/t2t-code-search:predict",
92101
},
93102
},
94103
}

code_search/kubeflow/components/nms-creator.jsonnet renamed to code_search/kubeflow/components/search-index-creator.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ local k = import "k.libsonnet";
22
local nms = import "nms.libsonnet";
33

44
local env = std.extVar("__ksonnet/environments");
5-
local params = std.extVar("__ksonnet/params").components["nms-creator"];
5+
local params = std.extVar("__ksonnet/params").components["search-index-creator"];
66

77
std.prune(k.core.v1.list.new(nms.parts(params, env).creator))

code_search/kubeflow/components/nms-server.jsonnet renamed to code_search/kubeflow/components/search-index-server.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ local k = import "k.libsonnet";
22
local nms = import "nms.libsonnet";
33

44
local env = std.extVar("__ksonnet/environments");
5-
local params = std.extVar("__ksonnet/params").components["nms-server"];
5+
local params = std.extVar("__ksonnet/params").components["search-index-server"];
66

77
std.prune(k.core.v1.list.new(nms.parts(params, env).server))

‎code_search/kubeflow/vendor/kubeflow/tf-serving@e95f94a1a97a0974ada734895d590b5ba565fa77/tf-serving.libsonnet‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@
122122
args: [
123123
"/usr/bin/tensorflow_model_server",
124124
"--port=9000",
125+
"--rest_api_port=8000",
125126
"--model_name=" + $.params.modelName,
126127
"--model_base_path=" + $.params.modelPath,
127128
],
@@ -342,7 +343,7 @@
342343
gcpParts:: $.parts {
343344
gcpEnv:: [
344345
if $.gcpParams.gcpCredentialSecretName != "" then
345-
{ name: "GOOGLE_APPLICATION_CREDENTIALS", value: "/secret/gcp-credentials/key.json" },
346+
{ name: "GOOGLE_APPLICATION_CREDENTIALS", value: "/secret/gcp-credentials/user-gcp-sa.json" },
346347
],
347348

348349
tfServingContainer: $.parts.tfServingContainer {

‎code_search/src/code_search/nmslib/search_server.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os
12
from flask import Flask, request, abort, jsonify, make_response, redirect
23

34

@@ -29,7 +30,8 @@ def init_routes(self):
2930

3031
@self.app.route('/')
3132
def index():
32-
return redirect('/index.html', code=302)
33+
redirect_path = os.environ.get('PUBLIC_URL', '') + '/index.html'
34+
return redirect(redirect_path, code=302)
3335

3436
@self.app.route('/ping')
3537
def ping():

‎code_search/src/ui/src/CodeSearchApi.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import request from 'superagent';
22

3-
const SEARCH_URL='/query';
3+
const SEARCH_URL=`${process.env.PUBLIC_URL}/query`;
44

55
function code_search_api(str) {
66
return request.get(SEARCH_URL).query({'q': str});

0 commit comments

Comments
 (0)