diff --git a/react-native/package.json b/react-native/package.json index e858114..3d879e4 100644 --- a/react-native/package.json +++ b/react-native/package.json @@ -19,11 +19,14 @@ "expo-auth-session": "~3.5.0", "expo-camera": "^12.1.2", "expo-font": "~10.0.4", - "expo-splash-screen": "~0.14.1", + "expo-image-picker": "~12.0.1", "expo-random": "~12.1.1", + "expo-splash-screen": "~0.14.1", "expo-status-bar": "~1.2.0", "expo-web-browser": "~10.1.0", + "form-data": "^4.0.0", "metro-react-native-babel-preset": "^0.69.0", + "mime": "^3.0.0", "native-base": "^3.3.7", "react": "17.0.1", "react-dom": "17.0.1", @@ -45,6 +48,7 @@ }, "devDependencies": { "@babel/core": "^7.12.9", + "@types/mime": "^2.0.3", "@types/react": "~17.0.21", "@types/react-native": "~0.64.12", "@types/react-native-dotenv": "^0.2.0", diff --git a/react-native/screens/TranslateScreen.tsx b/react-native/screens/TranslateScreen.tsx index ea7499b..29662e8 100644 --- a/react-native/screens/TranslateScreen.tsx +++ b/react-native/screens/TranslateScreen.tsx @@ -9,235 +9,270 @@ import useFonts from '../hooks/useFonts' import SwipeUpDown from 'react-native-swipe-up-down'; import BottomDrawer from '../components/BottomDrawer'; import { useToast, Box } from 'native-base'; +import mime from "mime"; +import * as ImagePicker from 'expo-image-picker'; /* TODO: - - 스크롤 내려가게 하기 (지금은 ScrollView의 스크롤이 안 먹음) - - low highlight 주기 (지금은 텍스트 높이만큼 background에 색 줘서 highlight) + - 스크롤 내려가게 하기 (지금은 ScrollView의 스크롤이 안 먹음) + - low highlight 주기 (지금은 텍스트 높이만큼 background에 색 줘서 highlight) */ export default function TranslateScreen({ navigation }: Navigation) { - const [hasPermission, setHasPermission] = useState(false); - const [fontsLoaded, SetFontsLoaded] = useState(false); - const [type, setType] = useState(Camera.Constants.Type.back); - const [camera, setCamera] = useState(null); - const [imageUri, setImageUri] = useState(""); - const [results, setResults] = useState({id: 0, summary: [], fullText: '', korean: ''}); - const [showFullText, setShowFullText] = useState(false); - const [isFullDrawer, setFullDrawer] = useState(false); - - const toast = useToast(); - - const LoadFontsAndRestoreToken = async () => { - await useFonts(); - }; - - useEffect(() => { - (async () => { - const { status } = await Camera.requestCameraPermissionsAsync(); - setHasPermission(status === 'granted'); - })(); - extractText - }, []); - - useEffect(() => { - if (results?.summary && results.summary.filter(item => item.highlight === true).length > 0) { - const message = "You can add a schedule to the calendar by clicking the highlighted text." - toast.show({ // Design according to mui toast guidelines (https://material.io/components/snackbars#anatomy) - render: () => { - return - {message} - ; - } - }); - } - }, [results]) - - // if (hasPermission === null) { - // return ; - // } - // else if (hasPermission === false) { - // return No access to camera! - // } - - if (!fontsLoaded) { - return ( - SetFontsLoaded(true)} - onError={() => {}} - /> - ); - } - - const takePicture = async () => { - if (camera) { - const data = await camera.takePictureAsync(null); - console.log(data.uri); - setImageUri(data.uri); - } - }; - - const extractText = (): void => { - // TODO: api - // TEST - setResults({ - id: 1, - summary: [ - {"id": 1, "content": "Buy Suyeon a delicious meal.", "highlight": false, registered: false}, - {"id": 2, "content": "The graduation ceremony will be held in the auditorium at 2 p.m. on February 14th.", "highlight": true, registered: false}, - {"id": 3, "content": "Parents and outsiders are not allowed to enter each class.", "highlight": true, registered: true}, - ], - fullText: "We wish you good health and happiness in your family in the hopeful new year of the 17th graduation ceremony in 2020. Thank you very much for supporting our educational activities and for your constant interest and cooperation in the midst of the 2020 COVID-19 crisis. The 17th graduation ceremony of Hwaam High School will be held in classrooms to prevent the spread of COVID-19. Parents and outsiders are not allowed to enter each class. Please understand that we cannot bring you together because it is an inevitable measure to prevent the spread of infectious diseases.", - korean: "2020학년도 제17회 졸업식 실시 안내 희망찬 새해를 맞이하여 학부모님의 가정에 건강과 행복이 함께 하시기를 기원합니다. 2020학년도 코로나19라는 위기 상황 속에서 본교 교육활동을 지지해주시고 변함없는 관심과 협조를 보내 주셔서 진심으로 감사드립니다. 화암고등학교 제17회 졸업식은 코로나19 확산 예방을 위해 각반 교실에서 진행합니다. 각반 교실에는 학부모 및 외부인의 출입이 불가능합니다. 감염병 확산 방지를 위한 불가피한 대책이니 함께 자리에 모시지 못함을 양해 부탁드립니다. 희망찬 내일을 향해 첫발을 내딛는 졸업생들에게 멀리서나마 축하와 격려를 보내주시기 바랍니다. -제17회 졸업식 안내- 1. 일시: 2021년 1월 5일(화) 10:00 2. 장소: 각반 교실 3: 내용: 각반 영상으로 졸업식 실시 * 졸업생 출입 가능 / 학부모 및 외부인 출입 불가 * 졸업생은 반드시 마스크 착용, 자가 진단 후 등교" - }) - } - - const handleFullText = (): void => { - setShowFullText(!showFullText); - } - - const saveResults = (): void => { - // TODO: api - Alert.alert("The scanned result was saved in ."); - } - - const closeResults = (): void => { - navigation.navigate('Home'); - } - - const retakePicture = (): void => { - setImageUri(''); - setResults({id: 0, summary: [], fullText: '', korean: ''}); - setShowFullText(false); - } - - return ( - - {/* After taking a picture */} - {imageUri ? ( - /* After taking a picture and press the check button */ - results?.summary && results?.fullText && results?.korean ? ( - - - } - itemFull={ - - } - onShowMini={() => setFullDrawer(false)} - onShowFull={() => setFullDrawer(true)} - animation="easeInEaseOut" - disableSwipeIcon - extraMarginTop={10} - swipeHeight={Dimensions.get('window').height*0.5} - /> - - ) : ( - /* After taking a picture, before OCR(pressing the check button) */ - <> - - - - - - - - ) - ) : ( - /* Before taking a picture, Camera ready */ - <> - setCamera(ref)}> - - { - setType( - type === Camera.Constants.Type.back - ? Camera.Constants.Type.front - : Camera.Constants.Type.back - ); - }}> - - - - - - - - - - - )} - - ); + const [hasPermission, setHasPermission] = useState(false); + const [fontsLoaded, SetFontsLoaded] = useState(false); + const [type, setType] = useState(Camera.Constants.Type.back); + const [camera, setCamera] = useState(null); + const [imageUri, setImageUri] = useState(''); + const [results, setResults] = useState({id: 0, summary: [], fullText: '', korean: ''}); + const [showFullText, setShowFullText] = useState(false); + const [isFullDrawer, setFullDrawer] = useState(false); + + const toast = useToast(); + + const LoadFontsAndRestoreToken = async () => { + await useFonts(); + }; + + useEffect(() => { + (async () => { + const { status } = await Camera.requestCameraPermissionsAsync(); + setHasPermission(status === 'granted'); + })(); + }, []); + + useEffect(() => { + if (imageUri) { + extractText + } + }, [imageUri]) + + useEffect(() => { + if (results?.summary && results.summary.filter(item => item.highlight === true).length > 0) { + const message = "You can add a schedule to the calendar by clicking the highlighted text." + toast.show({ // Design according to mui toast guidelines (https://material.io/components/snackbars#anatomy) + render: () => { + return + {message} + ; + } + }); + } + }, [results]) + + // DEV TEST + // if (hasPermission === null) { + // return ; + // } + // else if (hasPermission === false) { + // return No access to camera! + // } + + if (!fontsLoaded) { + return ( + SetFontsLoaded(true)} + onError={() => {}} + /> + ); + } + + const takePicture = async () => { + if (camera) { + const data = await camera.takePictureAsync(null); + setImageUri(data.uri); + } + }; + + const pickImage = async () => { + // No permissions request is necessary for launching the image library + let result = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ImagePicker.MediaTypeOptions.All, + allowsEditing: true, + aspect: [4, 3], + quality: 1, + }); + + if (!result.cancelled) { + setImageUri(result.uri); + } + }; + + const extractText = (): void => { + if (imageUri) { + let FormData = require('form-data'); + const formdata = new FormData(); + formdata.append("uploadfile", { + uri : imageUri, + type: mime.getType(imageUri), + name: imageUri.split("/").pop() + }); + + fetch("http://localhost:8080/notice/ocr", { + method: 'POST', + body: formdata, + redirect: 'follow' + }) + .then(response => response.text()) // TODO: response.json() + .then(data => console.log(data)) // TODO: setResults(data) + .catch(error => console.log('error', error)); + + // TEST + setResults({ + id: 1, + summary: [ + {"id": 1, "content": "Buy Suyeon a delicious meal.", "highlight": false, registered: false}, + {"id": 2, "content": "The graduation ceremony will be held in the auditorium at 2 p.m. on February 14th.", "highlight": true, registered: false}, + {"id": 3, "content": "Parents and outsiders are not allowed to enter each class.", "highlight": true, registered: true}, + ], + fullText: "We wish you good health and happiness in your family in the hopeful new year of the 17th graduation ceremony in 2020. Thank you very much for supporting our educational activities and for your constant interest and cooperation in the midst of the 2020 COVID-19 crisis. The 17th graduation ceremony of Hwaam High School will be held in classrooms to prevent the spread of COVID-19. Parents and outsiders are not allowed to enter each class. Please understand that we cannot bring you together because it is an inevitable measure to prevent the spread of infectious diseases.", + korean: "2020학년도 제17회 졸업식 실시 안내 희망찬 새해를 맞이하여 학부모님의 가정에 건강과 행복이 함께 하시기를 기원합니다. 2020학년도 코로나19라는 위기 상황 속에서 본교 교육활동을 지지해주시고 변함없는 관심과 협조를 보내 주셔서 진심으로 감사드립니다. 화암고등학교 제17회 졸업식은 코로나19 확산 예방을 위해 각반 교실에서 진행합니다. 각반 교실에는 학부모 및 외부인의 출입이 불가능합니다. 감염병 확산 방지를 위한 불가피한 대책이니 함께 자리에 모시지 못함을 양해 부탁드립니다. 희망찬 내일을 향해 첫발을 내딛는 졸업생들에게 멀리서나마 축하와 격려를 보내주시기 바랍니다. -제17회 졸업식 안내- 1. 일시: 2021년 1월 5일(화) 10:00 2. 장소: 각반 교실 3: 내용: 각반 영상으로 졸업식 실시 * 졸업생 출입 가능 / 학부모 및 외부인 출입 불가 * 졸업생은 반드시 마스크 착용, 자가 진단 후 등교" + }) + } + } + + const handleFullText = (): void => { + setShowFullText(!showFullText); + } + + const saveResults = (): void => { + // TODO: api + Alert.alert("The scanned result was saved in ."); + } + + const closeResults = (): void => { + navigation.navigate('Home'); + } + + const retakePicture = (): void => { + setImageUri(''); + setResults({id: 0, summary: [], fullText: '', korean: ''}); + setShowFullText(false); + } + + return ( + + {/* After taking a picture */} + {imageUri ? ( + /* After taking a picture and press the check button */ + results?.summary && results?.fullText && results?.korean ? ( + + + } + itemFull={ + + } + onShowMini={() => setFullDrawer(false)} + onShowFull={() => setFullDrawer(true)} + animation="easeInEaseOut" + disableSwipeIcon + extraMarginTop={10} + swipeHeight={Dimensions.get('window').height*0.5} + /> + + ) : ( + /* After taking a picture, before OCR(pressing the check button) */ + <> + + + + + + + + ) + ) : ( + /* Before taking a picture, Camera ready */ + <> + setCamera(ref)}> + + + + + + + + + + { + setType( + type === Camera.Constants.Type.back + ? Camera.Constants.Type.front + : Camera.Constants.Type.back + ); + }}> + + + + + )} + + ); } const styles = StyleSheet.create({ - container: { - flex: 1, - }, - camera: { - flex: 4, - }, - cameraContainer: { - flex: 1, - backgroundColor: 'transparent', - flexDirection: 'row', - margin: 20, - }, - reverseButton: { - flex: 0.1, - alignSelf: 'flex-end', - alignItems: 'center', - }, - buttonContainer: { - flex: 1, - flexDirection: 'row', - justifyContent: 'center', - alignItems: 'center', - backgroundColor: '#000' - }, - circleButton: { - borderRadius: 48, - height: 64, - width: 64, - flexDirection: 'row', - justifyContent: 'center', - alignItems: 'center', - }, - primaryBackground: { - backgroundColor: theme.colors.primary - }, - grayBackground: { - backgroundColor: theme.colors.gray - }, - whiteBackground: { - backgroundColor: '#fff', - }, - innerCircle: { - borderRadius: 48, - padding: 8, - height: 56, - width: 56, - borderWidth: 2 - } + container: { + flex: 1, + }, + camera: { + flex: 4, + }, + cameraContainer: { + flex: 1, + backgroundColor: 'transparent', + flexDirection: 'row', + margin: 20, + }, + buttonContainer: { + flex: 1, + flexDirection: 'row', + alignItems: 'center', + paddingHorizontal: 24, + backgroundColor: '#000' + }, + circleButton: { + borderRadius: 48, + height: 64, + width: 64, + flexDirection: 'row', + justifyContent: 'center', + alignItems: 'center', + }, + primaryBackground: { + backgroundColor: theme.colors.primary + }, + grayBackground: { + backgroundColor: theme.colors.gray + }, + whiteBackground: { + backgroundColor: '#fff', + }, + innerCircle: { + borderRadius: 48, + padding: 8, + height: 56, + width: 56, + borderWidth: 2 + } }); diff --git a/react-native/yarn.lock b/react-native/yarn.lock index e16b59a..e7fee62 100644 --- a/react-native/yarn.lock +++ b/react-native/yarn.lock @@ -2382,6 +2382,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.179.tgz#490ec3288088c91295780237d2497a3aa9dfb5c5" integrity sha512-uwc1x90yCKqGcIOAT6DwOSuxnrAbpkdPsUOZtwrXb4D/6wZs+6qG7QnIawDuZWg0sWpxl+ltIKCaLoMlna678w== +"@types/mime@^2.0.3": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.3.tgz#c893b73721db73699943bfc3653b1deb7faa4a3a" + integrity sha512-Jus9s4CDbqwocc5pOAnh8ShfrnMcPHuJYzVcSUU7lrh8Ni5HuIqX3oilL86p3dlTrk0LzHRCgA/GQ7uNCw6l2Q== + "@types/node@*": version "17.0.18" resolved "https://registry.npmjs.org/@types/node/-/node-17.0.18.tgz" @@ -2632,6 +2637,11 @@ async@^2.4.0: dependencies: lodash "^4.17.14" +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + at-least-node@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz" @@ -3089,6 +3099,13 @@ colors@^1.1.2: resolved "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + command-exists@^1.2.8: version "1.2.9" resolved "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz" @@ -3358,6 +3375,11 @@ define-property@^2.0.2: is-descriptor "^1.0.2" isobject "^3.0.1" +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + denodeify@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz" @@ -3638,6 +3660,14 @@ expo-font@~10.0.4, expo-font@~10.0.5: dependencies: fontfaceobserver "^2.1.0" +expo-image-picker@~12.0.1: + version "12.0.2" + resolved "https://registry.yarnpkg.com/expo-image-picker/-/expo-image-picker-12.0.2.tgz#4789c5419208ff8ede557784368a7f18f3878c30" + integrity sha512-rAoNGtofV5cg3UN+cdIGDVfbMvbutBX6uNV7jCIEw/WZxZlbW+R7HC8l5lGDFtoLVqcpkHE/6JpAXvESxVSAOA== + dependencies: + "@expo/config-plugins" "^4.0.2" + uuid "7.0.2" + expo-keep-awake@~10.0.2: version "10.0.2" resolved "https://registry.npmjs.org/expo-keep-awake/-/expo-keep-awake-10.0.2.tgz" @@ -3912,6 +3942,15 @@ for-in@^1.0.2: resolved "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz" @@ -5180,6 +5219,18 @@ mime-db@1.51.0, "mime-db@>= 1.43.0 < 2": resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz" integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime-types@^2.1.27, mime-types@~2.1.34: version "2.1.34" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz" @@ -5197,6 +5248,11 @@ mime@^2.4.1, mime@^2.4.4: resolved "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz" integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg== +mime@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" + integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== + mimic-fn@^1.0.0: version "1.2.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz" @@ -6926,6 +6982,11 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-7.0.2.tgz#7ff5c203467e91f5e0d85cfcbaaf7d2ebbca9be6" + integrity sha512-vy9V/+pKG+5ZTYKf+VcphF5Oc6EFiu3W8Nv3P3zIh0EqVI80ZxOzuPfe9EHjkFNvf8+xuTHVeei4Drydlx4zjw== + uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz"